session-steward 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +100 -3
- package/bin/session-steward-cli.mjs +4 -0
- package/bin/session-steward-mcp.mjs +65 -0
- package/dist/assets/index-C94A1O5c.js +9 -0
- package/dist/assets/index-CXq8Tw8T.css +2 -0
- package/dist/index.html +2 -2
- package/lib/cli.mjs +102 -2
- package/lib/mcp.mjs +607 -0
- package/lib/providers/claude-code/events.mjs +17 -1
- package/lib/providers/claude-code/index.mjs +2 -0
- package/lib/providers/claude-code/store.mjs +4 -0
- package/lib/providers/claude-code/tokens.mjs +174 -0
- package/lib/providers/codex/events.mjs +17 -1
- package/lib/providers/codex/index.mjs +2 -0
- package/lib/providers/codex/store.mjs +14 -2
- package/lib/providers/codex/tokens.mjs +317 -0
- package/lib/server.mjs +30 -0
- package/lib/session-events.mjs +4 -0
- package/lib/session-token-cache.mjs +76 -0
- package/lib/session-tokens.mjs +73 -0
- package/package.json +9 -2
- package/dist/assets/index-BOACkzUI.js +0 -9
- package/dist/assets/index-DFAWGcgb.css +0 -2
|
@@ -424,6 +424,10 @@ function filterRecords(records, options) {
|
|
|
424
424
|
if (options.archiveStatus === "active" && record.archived) return false;
|
|
425
425
|
if (options.archiveStatus === "archived" && !record.archived) return false;
|
|
426
426
|
if (options.inactiveBeforeMs && (!record.updatedAtMs || record.updatedAtMs >= options.inactiveBeforeMs)) return false;
|
|
427
|
+
if (Number.isFinite(options.minimumTranscriptBytes)
|
|
428
|
+
&& options.minimumTranscriptBytes > 0
|
|
429
|
+
&& (!Number.isFinite(record.transcriptBytes)
|
|
430
|
+
|| record.transcriptBytes < options.minimumTranscriptBytes)) return false;
|
|
427
431
|
if (options.workspace !== undefined && record.cwd !== options.workspace) return false;
|
|
428
432
|
if (search && !`${record.displayName} ${record.searchText} ${record.id} ${record.cwd} ${record.surface}`.toLowerCase().includes(search)) return false;
|
|
429
433
|
return true;
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { addTokenTotals, createTokenTotals, summarizeSessionTokens } from "../../session-tokens.mjs";
|
|
2
|
+
import { readCachedTokens, readFileStamp, writeCachedTokens } from "../../session-token-cache.mjs";
|
|
3
|
+
import { visitJsonlSnapshotEntries } from "../../storage/jsonl.mjs";
|
|
4
|
+
import { getSessionRecord } from "./store.mjs";
|
|
5
|
+
|
|
6
|
+
const SYNTHETIC_MODEL = "<synthetic>";
|
|
7
|
+
|
|
8
|
+
// Claude Code reports usage per request rather than as a running total, so there
|
|
9
|
+
// is no counter to reconcile. The catch is the opposite one: a single response is
|
|
10
|
+
// written as several records, one per content block, and every copy repeats the
|
|
11
|
+
// same usage in full. Summing records instead of requests inflates a session by
|
|
12
|
+
// roughly 2.3x.
|
|
13
|
+
function readCount(value) {
|
|
14
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.trunc(value) : 0;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
// Anthropic reports cached tokens *beside* the input count, where Codex reports
|
|
18
|
+
// them inside it. `input_tokens` is already the uncached remainder, so nothing is
|
|
19
|
+
// subtracted here — doing so is what would drive the fresh-input slice negative.
|
|
20
|
+
function normalizeUsage(value) {
|
|
21
|
+
if (!value || typeof value !== "object") return null;
|
|
22
|
+
|
|
23
|
+
const cachedInput = readCount(value.cache_read_input_tokens);
|
|
24
|
+
const cacheWrites = readCount(value.cache_creation_input_tokens);
|
|
25
|
+
const freshInput = readCount(value.input_tokens);
|
|
26
|
+
const output = readCount(value.output_tokens);
|
|
27
|
+
|
|
28
|
+
return {
|
|
29
|
+
cachedInput,
|
|
30
|
+
cacheWrites,
|
|
31
|
+
freshInput,
|
|
32
|
+
output,
|
|
33
|
+
// Claude Code does not report a reasoning figure; thinking is billed as
|
|
34
|
+
// ordinary output and cannot be separated from it.
|
|
35
|
+
reasoning: 0,
|
|
36
|
+
total: cachedInput + cacheWrites + freshInput + output,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function createClaudeTokenCollector() {
|
|
41
|
+
// One entry per request, not per record. Repeats collapse onto the largest
|
|
42
|
+
// copy: a retried or superseded write reports zeros, and a streaming partial
|
|
43
|
+
// reports less than the finished response.
|
|
44
|
+
const requests = new Map();
|
|
45
|
+
let compactions = 0;
|
|
46
|
+
let observedRecords = 0;
|
|
47
|
+
let sessionId = null;
|
|
48
|
+
let sidechainRequests = 0;
|
|
49
|
+
let syntheticRecords = 0;
|
|
50
|
+
|
|
51
|
+
return {
|
|
52
|
+
record(recordValue) {
|
|
53
|
+
if (!recordValue || typeof recordValue !== "object") return;
|
|
54
|
+
|
|
55
|
+
if (recordValue.subtype === "compact_boundary") compactions += 1;
|
|
56
|
+
if (sessionId === null && typeof recordValue.sessionId === "string") {
|
|
57
|
+
sessionId = recordValue.sessionId;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const message = recordValue.message;
|
|
61
|
+
if (!message || typeof message !== "object") return;
|
|
62
|
+
|
|
63
|
+
const usage = normalizeUsage(message.usage);
|
|
64
|
+
if (!usage) return;
|
|
65
|
+
observedRecords += 1;
|
|
66
|
+
|
|
67
|
+
// Synthetic entries stand in for locally generated messages. They carry no
|
|
68
|
+
// usage and no request id, and would otherwise open a model row of zeros.
|
|
69
|
+
if (message.model === SYNTHETIC_MODEL) {
|
|
70
|
+
syntheticRecords += 1;
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const key = recordValue.requestId ?? message.id;
|
|
75
|
+
if (typeof key !== "string") return;
|
|
76
|
+
|
|
77
|
+
const previous = requests.get(key);
|
|
78
|
+
if (previous && previous.usage.total >= usage.total) return;
|
|
79
|
+
if (!previous && recordValue.isSidechain) sidechainRequests += 1;
|
|
80
|
+
|
|
81
|
+
requests.set(key, { model: message.model ?? "Unknown", sidechain: Boolean(recordValue.isSidechain), usage });
|
|
82
|
+
},
|
|
83
|
+
|
|
84
|
+
result() {
|
|
85
|
+
const totals = createTokenTotals();
|
|
86
|
+
const modelTotals = new Map();
|
|
87
|
+
|
|
88
|
+
for (const { model, usage } of requests.values()) {
|
|
89
|
+
addTokenTotals(totals, usage);
|
|
90
|
+
if (!modelTotals.has(model)) modelTotals.set(model, createTokenTotals());
|
|
91
|
+
addTokenTotals(modelTotals.get(model), usage);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
return {
|
|
95
|
+
available: requests.size > 0,
|
|
96
|
+
byModel: [...modelTotals]
|
|
97
|
+
.map(([model, modelSpend]) => ({ model, totals: modelSpend }))
|
|
98
|
+
.sort((left, right) => right.totals.total - left.totals.total),
|
|
99
|
+
cacheWriteUnderflow: false,
|
|
100
|
+
compactions,
|
|
101
|
+
countedRequests: requests.size,
|
|
102
|
+
observedRecords,
|
|
103
|
+
sessionId,
|
|
104
|
+
sidechainRequests,
|
|
105
|
+
syntheticRecords,
|
|
106
|
+
totals,
|
|
107
|
+
};
|
|
108
|
+
},
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export async function collectClaudeSessionTokens(filePath, { maxLineBytes, signal } = {}) {
|
|
113
|
+
const collector = createClaudeTokenCollector();
|
|
114
|
+
const { complete, snapshotBytes } = await visitJsonlSnapshotEntries(
|
|
115
|
+
filePath,
|
|
116
|
+
({ parsed }) => {
|
|
117
|
+
// A closed panel should not leave a large transcript being scanned.
|
|
118
|
+
if (signal?.aborted) return false;
|
|
119
|
+
if (parsed && typeof parsed === "object") collector.record(parsed);
|
|
120
|
+
return true;
|
|
121
|
+
},
|
|
122
|
+
maxLineBytes === undefined ? {} : { maxLineBytes },
|
|
123
|
+
);
|
|
124
|
+
|
|
125
|
+
return { ...collector.result(), complete, snapshotBytes };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Reading the timeline already streams every record of the same file, so the
|
|
129
|
+
// count rides along with that pass instead of paying for a second one.
|
|
130
|
+
export async function createSessionTokenScan({ record, signal }) {
|
|
131
|
+
if (!record?.rolloutPath) return null;
|
|
132
|
+
const stamp = await readFileStamp(record.rolloutPath);
|
|
133
|
+
const cached = readCachedTokens("summary", record.rolloutPath, stamp);
|
|
134
|
+
if (cached !== undefined) return { cached, record() {}, summarize: () => cached };
|
|
135
|
+
|
|
136
|
+
const collector = createClaudeTokenCollector();
|
|
137
|
+
|
|
138
|
+
return {
|
|
139
|
+
cached: null,
|
|
140
|
+
record(value) {
|
|
141
|
+
collector.record(value);
|
|
142
|
+
},
|
|
143
|
+
summarize({ complete }) {
|
|
144
|
+
const summary = summarizeSessionTokens({ ...collector.result(), complete });
|
|
145
|
+
if (signal?.aborted || complete === false) return summary;
|
|
146
|
+
return writeCachedTokens("summary", record.rolloutPath, stamp, summary);
|
|
147
|
+
},
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export async function readSessionTokens({ claudeHome, desktopDataHome, id, maxLineBytes, signal }) {
|
|
152
|
+
const record = await getSessionRecord({ claudeHome, desktopDataHome, id });
|
|
153
|
+
if (!record) return null;
|
|
154
|
+
if (!record.rolloutPath) return { available: false, reason: "no-transcript-path" };
|
|
155
|
+
|
|
156
|
+
const stamp = await readFileStamp(record.rolloutPath);
|
|
157
|
+
const cached = readCachedTokens("summary", record.rolloutPath, stamp);
|
|
158
|
+
if (cached !== undefined) return cached;
|
|
159
|
+
|
|
160
|
+
let collected;
|
|
161
|
+
try {
|
|
162
|
+
collected = await collectClaudeSessionTokens(record.rolloutPath, { maxLineBytes, signal });
|
|
163
|
+
} catch (error) {
|
|
164
|
+
// A transcript can be deleted between the listing and the read. That is an
|
|
165
|
+
// answer about the session, not a failure of the server, and the timeline
|
|
166
|
+
// reader already treats it as one.
|
|
167
|
+
if (error?.code === "ENOENT") return { available: false, reason: "transcript-missing" };
|
|
168
|
+
throw error;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const summary = summarizeSessionTokens(collected);
|
|
172
|
+
if (signal?.aborted || collected.complete === false) return summary;
|
|
173
|
+
return writeCachedTokens("summary", record.rolloutPath, stamp, summary);
|
|
174
|
+
}
|
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
createSessionEventsResult,
|
|
8
8
|
finalizeSessionEventComposition,
|
|
9
9
|
SESSION_EVENT_KIND,
|
|
10
|
+
SESSION_EVENT_READ_MODE,
|
|
10
11
|
SESSION_EVENT_REASON,
|
|
11
12
|
} from "../../session-events.mjs";
|
|
12
13
|
import {
|
|
@@ -16,6 +17,7 @@ import {
|
|
|
16
17
|
} from "../../session-event-reader.mjs";
|
|
17
18
|
import { visitJsonlSnapshotEntries } from "../../storage/jsonl.mjs";
|
|
18
19
|
import { getSessionRecord, loadSessionStore } from "./store.mjs";
|
|
20
|
+
import { createSessionTokenScan } from "./tokens.mjs";
|
|
19
21
|
|
|
20
22
|
const PROVIDER_ID = "codex";
|
|
21
23
|
const RECORD_CLASSIFICATION = Object.freeze({
|
|
@@ -325,12 +327,15 @@ function outputText(payload) {
|
|
|
325
327
|
return contentText(payload.output ?? payload.content ?? payload.stderr ?? payload.stdout).trim() || null;
|
|
326
328
|
}
|
|
327
329
|
|
|
328
|
-
function emptyResult({ cwd = null, origin = null, reason }) {
|
|
330
|
+
function emptyResult({ counted = false, cwd = null, origin = null, reason }) {
|
|
329
331
|
return createSessionEventsResult({
|
|
330
332
|
coverage: createSessionEventCoverage(),
|
|
331
333
|
events: [],
|
|
332
334
|
header: createSessionEventHeader({ cwd, origin, provider: PROVIDER_ID }),
|
|
333
335
|
reason,
|
|
336
|
+
// The reason the timeline is empty is the same reason there is no count:
|
|
337
|
+
// no transcript to read. Saying so beats leaving the field to guess.
|
|
338
|
+
tokens: counted ? { available: false, reason } : null,
|
|
334
339
|
window: {
|
|
335
340
|
complete: true,
|
|
336
341
|
end: null,
|
|
@@ -352,6 +357,7 @@ export async function readSessionEvents({
|
|
|
352
357
|
maxLineBytes,
|
|
353
358
|
mode,
|
|
354
359
|
signal,
|
|
360
|
+
tokens = false,
|
|
355
361
|
}) {
|
|
356
362
|
const record = await findSessionRecord({ codexHome, id });
|
|
357
363
|
if (!record) return null;
|
|
@@ -359,12 +365,19 @@ export async function readSessionEvents({
|
|
|
359
365
|
const origin = record.recordSource ?? null;
|
|
360
366
|
if (!record.rolloutPath) {
|
|
361
367
|
return emptyResult({
|
|
368
|
+
counted: tokens,
|
|
362
369
|
cwd: record.cwd || null,
|
|
363
370
|
origin,
|
|
364
371
|
reason: SESSION_EVENT_REASON.NO_TRANSCRIPT_PATH,
|
|
365
372
|
});
|
|
366
373
|
}
|
|
367
374
|
|
|
375
|
+
// A preview stops as soon as it has enough events, so a count taken from it
|
|
376
|
+
// would be of part of the file while reading as the whole.
|
|
377
|
+
const tokenScan = tokens && mode !== SESSION_EVENT_READ_MODE.PREVIEW
|
|
378
|
+
? await createSessionTokenScan({ codexHome, maxLineBytes, record, signal })
|
|
379
|
+
: null;
|
|
380
|
+
|
|
368
381
|
const coverage = createSessionEventCoverage();
|
|
369
382
|
const summary = createSessionEventSummary();
|
|
370
383
|
const composition = createSessionEventComposition();
|
|
@@ -778,6 +791,7 @@ export async function readSessionEvents({
|
|
|
778
791
|
}
|
|
779
792
|
|
|
780
793
|
composition[compositionSegment(entry.parsed)] += entry.bytes;
|
|
794
|
+
tokenScan?.record(entry.parsed);
|
|
781
795
|
const result = handleRecord(entry.parsed, entry.index);
|
|
782
796
|
coverage[result.classification] += 1;
|
|
783
797
|
if (result.duplicate) coverage.duplicates += 1;
|
|
@@ -791,6 +805,7 @@ export async function readSessionEvents({
|
|
|
791
805
|
} catch (error) {
|
|
792
806
|
if (error?.code === "ENOENT") {
|
|
793
807
|
return emptyResult({
|
|
808
|
+
counted: tokens,
|
|
794
809
|
cwd: record.cwd || null,
|
|
795
810
|
origin,
|
|
796
811
|
reason: SESSION_EVENT_REASON.TRANSCRIPT_MISSING,
|
|
@@ -811,6 +826,7 @@ export async function readSessionEvents({
|
|
|
811
826
|
: null,
|
|
812
827
|
composition: finalizeSessionEventComposition(composition, read.snapshotBytes),
|
|
813
828
|
summary,
|
|
829
|
+
tokens: tokenScan ? tokenScan.summarize(read) : null,
|
|
814
830
|
window: readState.window(read),
|
|
815
831
|
});
|
|
816
832
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readSessionEvents } from "./events.mjs";
|
|
2
|
+
import { readSessionTokens } from "./tokens.mjs";
|
|
2
3
|
import {
|
|
3
4
|
assertDeepCleanupSupported,
|
|
4
5
|
diagnoseStorageCompatibility,
|
|
@@ -40,6 +41,7 @@ export const codexProvider = Object.freeze({
|
|
|
40
41
|
planSessionDeletion,
|
|
41
42
|
preflightSessionDeletion,
|
|
42
43
|
readSessionEvents,
|
|
44
|
+
readSessionTokens,
|
|
43
45
|
restoreSessionDeletionBackup,
|
|
44
46
|
verifySessionDeletion,
|
|
45
47
|
});
|
|
@@ -1534,6 +1534,7 @@ export async function listSessions({
|
|
|
1534
1534
|
inactiveBeforeMs = null,
|
|
1535
1535
|
includeInternals = false,
|
|
1536
1536
|
includeSupporting = false,
|
|
1537
|
+
minimumTranscriptBytes = null,
|
|
1537
1538
|
page = 1,
|
|
1538
1539
|
pageSize = DEFAULT_PAGE_SIZE,
|
|
1539
1540
|
refresh = false,
|
|
@@ -1548,7 +1549,12 @@ export async function listSessions({
|
|
|
1548
1549
|
: DEFAULT_PAGE_SIZE;
|
|
1549
1550
|
const requestedPage = Number.isFinite(page) ? Math.max(1, Math.trunc(page)) : 1;
|
|
1550
1551
|
const resolvedSort = SESSION_SORTS.has(sort) ? sort : "updated";
|
|
1551
|
-
|
|
1552
|
+
const resolvedMinimumTranscriptBytes = Number.isFinite(minimumTranscriptBytes)
|
|
1553
|
+
&& minimumTranscriptBytes > 0
|
|
1554
|
+
? Math.trunc(minimumTranscriptBytes)
|
|
1555
|
+
: null;
|
|
1556
|
+
const needsSizeIndex = resolvedSort === "size" || resolvedMinimumTranscriptBytes !== null;
|
|
1557
|
+
if (paths.stateDatabases.length === 1 && !needsSizeIndex && !forceUnion) {
|
|
1552
1558
|
const database = paths.stateDatabases[0];
|
|
1553
1559
|
const conditions = getSessionConditions(database, {
|
|
1554
1560
|
archiveStatus, inactiveBeforeMs, includeInternals, includeSupporting, search, workspace,
|
|
@@ -1583,6 +1589,7 @@ export async function listSessions({
|
|
|
1583
1589
|
const ordered = [];
|
|
1584
1590
|
const seenIds = paths.stateDatabases.length > 1 ? new Set() : null;
|
|
1585
1591
|
const compactSizeIds = resolvedSort === "size" && paths.stateDatabases.length === 1;
|
|
1592
|
+
const sizes = needsSizeIndex ? await getSessionSizeIndex(paths, { refresh }) : null;
|
|
1586
1593
|
let compareSortRows = null;
|
|
1587
1594
|
for (const database of paths.stateDatabases) {
|
|
1588
1595
|
const conditions = getSessionConditions(database, {
|
|
@@ -1599,6 +1606,12 @@ export async function listSessions({
|
|
|
1599
1606
|
const id = String(row.id);
|
|
1600
1607
|
if (seenIds?.has(id)) continue;
|
|
1601
1608
|
seenIds?.add(id);
|
|
1609
|
+
if (resolvedMinimumTranscriptBytes !== null) {
|
|
1610
|
+
const transcriptBytes = sizes.get(id);
|
|
1611
|
+
if (!Number.isFinite(transcriptBytes) || transcriptBytes < resolvedMinimumTranscriptBytes) {
|
|
1612
|
+
continue;
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1602
1615
|
if (compactSizeIds) {
|
|
1603
1616
|
ordered.push(id);
|
|
1604
1617
|
continue;
|
|
@@ -1609,7 +1622,6 @@ export async function listSessions({
|
|
|
1609
1622
|
}
|
|
1610
1623
|
}
|
|
1611
1624
|
if (resolvedSort === "size") {
|
|
1612
|
-
const sizes = await getSessionSizeIndex(paths, { refresh });
|
|
1613
1625
|
ordered.sort((left, right) => compareSessionIdsBySize(
|
|
1614
1626
|
compactSizeIds ? left : left.id,
|
|
1615
1627
|
compactSizeIds ? right : right.id,
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
import { addTokenTotals, createTokenTotals, summarizeSessionTokens } from "../../session-tokens.mjs";
|
|
2
|
+
import { readCachedTokens, readFileStamp, signatureBytes, writeCachedTokens } from "../../session-token-cache.mjs";
|
|
3
|
+
import { visitJsonlSnapshotEntries } from "../../storage/jsonl.mjs";
|
|
4
|
+
import { getSessionRecord, loadSessionStore } from "./store.mjs";
|
|
5
|
+
|
|
6
|
+
const PAYLOAD_TYPE = "token_count";
|
|
7
|
+
|
|
8
|
+
// Codex reports a running total plus the cost of the turn that produced it. The
|
|
9
|
+
// running total is not a session total: subagents inherit the parent thread's
|
|
10
|
+
// counter, resume restarts it, and the same figures are re-emitted when nothing
|
|
11
|
+
// advanced. Only `last_token_usage` on an event whose running total *moved* is a
|
|
12
|
+
// turn that belongs to this session.
|
|
13
|
+
function readCount(value) {
|
|
14
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.trunc(value) : 0;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
// Codex counts cached tokens inside `input_tokens`; whether cache writes are
|
|
18
|
+
// also inside it is unconfirmed, because the field is zero everywhere we can
|
|
19
|
+
// observe. Subtracting assumes they are. The clamp keeps a wrong assumption from
|
|
20
|
+
// reaching the UI as a negative slice, and reports itself so it can be fixed
|
|
21
|
+
// from evidence rather than replaced with a second guess.
|
|
22
|
+
function normalizeUsage(value) {
|
|
23
|
+
if (!value || typeof value !== "object") return null;
|
|
24
|
+
|
|
25
|
+
const input = readCount(value.input_tokens);
|
|
26
|
+
const cachedInput = readCount(value.cached_input_tokens);
|
|
27
|
+
const cacheWrites = readCount(value.cache_write_input_tokens);
|
|
28
|
+
const output = readCount(value.output_tokens);
|
|
29
|
+
const underflows = input - cachedInput - cacheWrites < 0;
|
|
30
|
+
const freshInput = Math.max(0, underflows ? input - cachedInput : input - cachedInput - cacheWrites);
|
|
31
|
+
// The bar is drawn from these four segments, so the total has to be their sum
|
|
32
|
+
// or the slices add up to more than the whole. Codex's own `total_tokens`
|
|
33
|
+
// agrees on every billable event in the corpus (53,685 of 53,685), but it
|
|
34
|
+
// cannot agree once the clamp above has moved a token, and the clamp exists
|
|
35
|
+
// precisely because that case is unobserved rather than impossible.
|
|
36
|
+
const total = freshInput + cachedInput + cacheWrites + output;
|
|
37
|
+
|
|
38
|
+
return {
|
|
39
|
+
// A turn with no input and no output has nothing to bill, whatever its
|
|
40
|
+
// reported total says.
|
|
41
|
+
billable: total > 0,
|
|
42
|
+
cachedInput,
|
|
43
|
+
cacheWrites,
|
|
44
|
+
freshInput,
|
|
45
|
+
output,
|
|
46
|
+
// Reasoning is part of output, never alongside it.
|
|
47
|
+
reasoning: Math.min(output, readCount(value.reasoning_output_tokens)),
|
|
48
|
+
// Codex advances its running counter by its own arithmetic, past events that
|
|
49
|
+
// report no components at all. Whether the counter moved is a question about
|
|
50
|
+
// its figure, so that figure is kept rather than recomputed.
|
|
51
|
+
reportedTotal: readCount(value.total_tokens),
|
|
52
|
+
total,
|
|
53
|
+
underflows,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function readModel(recordValue, payload) {
|
|
58
|
+
if (recordValue.type === "session_meta") return payload?.model ?? null;
|
|
59
|
+
if (recordValue.type === "turn_context") return payload?.model ?? recordValue.model ?? null;
|
|
60
|
+
if (payload?.type === "thread_settings_applied") return payload.thread_settings?.model ?? null;
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function isSubagentSource(source) {
|
|
65
|
+
return Boolean(source) && typeof source === "object" && "subagent" in source;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// A turn is identified by two numbers: the counter it advanced to, and what it
|
|
69
|
+
// cost. `signatures` records those as plain number arrays — 16 bytes a turn,
|
|
70
|
+
// where the turn objects they replace cost 179 — and `prefix` plays a parent's
|
|
71
|
+
// recording back against a fork as the fork streams, so the fork retains nothing
|
|
72
|
+
// at all. Neither is on by default; a plain session needs neither.
|
|
73
|
+
export function createCodexTokenCollector({ prefix = null, signatures = false } = {}) {
|
|
74
|
+
const totals = createTokenTotals();
|
|
75
|
+
const modelTotals = new Map();
|
|
76
|
+
const inherited = prefix ? createTokenTotals() : null;
|
|
77
|
+
const own = prefix ? createTokenTotals() : null;
|
|
78
|
+
const ownModels = prefix ? new Map() : null;
|
|
79
|
+
const runningTotals = signatures ? [] : null;
|
|
80
|
+
const turnTotals = signatures ? [] : null;
|
|
81
|
+
let cacheWriteUnderflow = false;
|
|
82
|
+
let countedTurns = 0;
|
|
83
|
+
let forkedFromId = null;
|
|
84
|
+
let inheritedTurns = 0;
|
|
85
|
+
let matchingPrefix = Boolean(prefix);
|
|
86
|
+
let model = null;
|
|
87
|
+
let observedEvents = 0;
|
|
88
|
+
let previousRunningTotal = null;
|
|
89
|
+
let sessionId = null;
|
|
90
|
+
let subagent = false;
|
|
91
|
+
|
|
92
|
+
function countTurn(usage, runningTotal) {
|
|
93
|
+
countedTurns += 1;
|
|
94
|
+
if (usage.underflows) cacheWriteUnderflow = true;
|
|
95
|
+
addTokenTotals(totals, usage);
|
|
96
|
+
|
|
97
|
+
const key = model ?? "Unknown";
|
|
98
|
+
if (!modelTotals.has(key)) modelTotals.set(key, createTokenTotals());
|
|
99
|
+
addTokenTotals(modelTotals.get(key), usage);
|
|
100
|
+
|
|
101
|
+
if (signatures) {
|
|
102
|
+
runningTotals.push(runningTotal);
|
|
103
|
+
turnTotals.push(usage.total);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
if (!prefix) return;
|
|
107
|
+
|
|
108
|
+
// The replay is a leading run, so the first turn that fails to match ends
|
|
109
|
+
// it. Everything from there on is this session's own work.
|
|
110
|
+
if (matchingPrefix) {
|
|
111
|
+
if (
|
|
112
|
+
inheritedTurns < prefix.runningTotals.length
|
|
113
|
+
&& prefix.runningTotals[inheritedTurns] === runningTotal
|
|
114
|
+
&& prefix.turnTotals[inheritedTurns] === usage.total
|
|
115
|
+
) {
|
|
116
|
+
inheritedTurns += 1;
|
|
117
|
+
addTokenTotals(inherited, usage);
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
matchingPrefix = false;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
addTokenTotals(own, usage);
|
|
125
|
+
if (!ownModels.has(key)) ownModels.set(key, createTokenTotals());
|
|
126
|
+
addTokenTotals(ownModels.get(key), usage);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
return {
|
|
130
|
+
record(recordValue) {
|
|
131
|
+
if (!recordValue || typeof recordValue !== "object") return;
|
|
132
|
+
const payload = recordValue.payload && typeof recordValue.payload === "object"
|
|
133
|
+
? recordValue.payload
|
|
134
|
+
: null;
|
|
135
|
+
|
|
136
|
+
if (recordValue.type === "session_meta" && payload && sessionId === null) {
|
|
137
|
+
sessionId = payload.id ?? null;
|
|
138
|
+
forkedFromId = payload.forked_from_id ?? null;
|
|
139
|
+
subagent = isSubagentSource(payload.source);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const nextModel = readModel(recordValue, payload);
|
|
143
|
+
if (nextModel) model = nextModel;
|
|
144
|
+
|
|
145
|
+
if (payload?.type !== PAYLOAD_TYPE) return;
|
|
146
|
+
observedEvents += 1;
|
|
147
|
+
|
|
148
|
+
// `info` is absent on some builds; the event carries nothing to bill.
|
|
149
|
+
const info = payload.info;
|
|
150
|
+
if (!info || typeof info !== "object") return;
|
|
151
|
+
|
|
152
|
+
const running = normalizeUsage(info.total_token_usage);
|
|
153
|
+
const turn = normalizeUsage(info.last_token_usage);
|
|
154
|
+
if (!running || !turn) return;
|
|
155
|
+
|
|
156
|
+
// Unchanged running total means the event was re-emitted, or carries
|
|
157
|
+
// context-size telemetry rather than a billed turn. Either way it is not
|
|
158
|
+
// new spend. A *decrease* is a resume resetting the counter, which is.
|
|
159
|
+
if (previousRunningTotal !== null && running.reportedTotal === previousRunningTotal) return;
|
|
160
|
+
|
|
161
|
+
previousRunningTotal = running.reportedTotal;
|
|
162
|
+
|
|
163
|
+
// Exceeding the context window makes Codex advance the running total to a
|
|
164
|
+
// synthetic figure while reporting a turn with no components. The advance
|
|
165
|
+
// is real bookkeeping; the turn behind it is not spend.
|
|
166
|
+
if (!turn.billable) return;
|
|
167
|
+
|
|
168
|
+
countTurn(turn, running.reportedTotal);
|
|
169
|
+
},
|
|
170
|
+
|
|
171
|
+
result() {
|
|
172
|
+
return {
|
|
173
|
+
available: countedTurns > 0,
|
|
174
|
+
byModel: [...modelTotals]
|
|
175
|
+
.map(([name, modelSpend]) => ({ model: name, totals: modelSpend }))
|
|
176
|
+
.sort((left, right) => right.totals.total - left.totals.total),
|
|
177
|
+
cacheWriteUnderflow,
|
|
178
|
+
countedTurns,
|
|
179
|
+
fork: prefix
|
|
180
|
+
? {
|
|
181
|
+
inherited,
|
|
182
|
+
inheritedTurns,
|
|
183
|
+
own,
|
|
184
|
+
ownByModel: [...ownModels]
|
|
185
|
+
.map(([name, modelSpend]) => ({ model: name, totals: modelSpend }))
|
|
186
|
+
.sort((left, right) => right.totals.total - left.totals.total),
|
|
187
|
+
ownTurns: countedTurns - inheritedTurns,
|
|
188
|
+
parentAvailable: true,
|
|
189
|
+
}
|
|
190
|
+
: null,
|
|
191
|
+
forkedFromId,
|
|
192
|
+
observedEvents,
|
|
193
|
+
sessionId,
|
|
194
|
+
signatures: signatures ? { runningTotals, turnTotals } : null,
|
|
195
|
+
subagent,
|
|
196
|
+
totals,
|
|
197
|
+
};
|
|
198
|
+
},
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
export async function collectCodexSessionTokens(filePath, { maxLineBytes, prefix, signatures, signal } = {}) {
|
|
203
|
+
const collector = createCodexTokenCollector({ prefix, signatures });
|
|
204
|
+
const { complete, snapshotBytes } = await visitJsonlSnapshotEntries(
|
|
205
|
+
filePath,
|
|
206
|
+
({ parsed }) => {
|
|
207
|
+
// A closed panel should not leave a large transcript being scanned.
|
|
208
|
+
if (signal?.aborted) return false;
|
|
209
|
+
if (parsed && typeof parsed === "object") collector.record(parsed);
|
|
210
|
+
return true;
|
|
211
|
+
},
|
|
212
|
+
maxLineBytes === undefined ? {} : { maxLineBytes },
|
|
213
|
+
);
|
|
214
|
+
|
|
215
|
+
return { ...collector.result(), complete, snapshotBytes };
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// A fork's rollout replays the parent's turns before its own, so the two files
|
|
219
|
+
// share a leading run of identical turns. Everything after that run is this
|
|
220
|
+
// session's own work; the run itself was already billed to the parent and must
|
|
221
|
+
// not be counted twice, in a rollup or on the session's own panel.
|
|
222
|
+
//
|
|
223
|
+
// There is no marker for the boundary inside the fork file. Anchoring at the
|
|
224
|
+
// last `session_meta` locates it 3 times in 44; a timestamp-gap test manages 40
|
|
225
|
+
// in 44, failing on the subagents that matter most. Comparing against the parent
|
|
226
|
+
// is the only exact method — so the parent is read first, and the fork is split
|
|
227
|
+
// as it streams rather than buffered and diffed afterwards.
|
|
228
|
+
async function findSessionRecord({ codexHome, id }) {
|
|
229
|
+
const record = await getSessionRecord({ codexHome, id });
|
|
230
|
+
if (record) return record;
|
|
231
|
+
return (await loadSessionStore({ codexHome })).recordsById.get(id) ?? null;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// A transcript can be deleted between the listing and the read. That is an
|
|
235
|
+
// answer about the session, not a failure of the server, and the timeline
|
|
236
|
+
// reader already treats it as one.
|
|
237
|
+
async function collectOrMissing(filePath, options) {
|
|
238
|
+
try {
|
|
239
|
+
return await collectCodexSessionTokens(filePath, options);
|
|
240
|
+
} catch (error) {
|
|
241
|
+
if (error?.code === "ENOENT") return null;
|
|
242
|
+
throw error;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// The parent is read only to locate the end of the replay, so its signatures
|
|
247
|
+
// are cached: a fork reopened, or several forks of one parent, read it once.
|
|
248
|
+
async function readParentSignatures({ codexHome, id, maxLineBytes, signal }) {
|
|
249
|
+
const record = await findSessionRecord({ codexHome, id });
|
|
250
|
+
if (!record?.rolloutPath) return null;
|
|
251
|
+
|
|
252
|
+
const stamp = await readFileStamp(record.rolloutPath);
|
|
253
|
+
const cached = readCachedTokens("signatures", record.rolloutPath, stamp);
|
|
254
|
+
if (cached !== undefined) return cached;
|
|
255
|
+
|
|
256
|
+
const parent = await collectOrMissing(record.rolloutPath, { maxLineBytes, signatures: true, signal });
|
|
257
|
+
const signatures = parent?.signatures ?? null;
|
|
258
|
+
if (signal?.aborted || parent?.complete === false) return signatures;
|
|
259
|
+
return writeCachedTokens("signatures", record.rolloutPath, stamp, signatures, signatureBytes(signatures));
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function summarizeCollected(collected) {
|
|
263
|
+
return summarizeSessionTokens(collected, {
|
|
264
|
+
fork: collected.fork,
|
|
265
|
+
// The file itself is the authority on whether this is a fork. If it says so
|
|
266
|
+
// and no parent was read, the total still carries the replay.
|
|
267
|
+
forkParentMissing: Boolean(collected.forkedFromId) && !collected.fork,
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// Reading the timeline already streams every record of the same file, so the
|
|
272
|
+
// count rides along with that pass instead of paying for a second one. The
|
|
273
|
+
// parent, when there is one, still has to be read first — and is cached.
|
|
274
|
+
export async function createSessionTokenScan({ codexHome, maxLineBytes, record, signal }) {
|
|
275
|
+
if (!record?.rolloutPath) return null;
|
|
276
|
+
const stamp = await readFileStamp(record.rolloutPath);
|
|
277
|
+
const cached = readCachedTokens("summary", record.rolloutPath, stamp);
|
|
278
|
+
if (cached !== undefined) return { cached, record() {}, summarize: () => cached };
|
|
279
|
+
|
|
280
|
+
const prefix = record.forkedFromId
|
|
281
|
+
? await readParentSignatures({ codexHome, id: record.forkedFromId, maxLineBytes, signal })
|
|
282
|
+
: null;
|
|
283
|
+
const collector = createCodexTokenCollector({ prefix });
|
|
284
|
+
|
|
285
|
+
return {
|
|
286
|
+
cached: null,
|
|
287
|
+
record(value) {
|
|
288
|
+
collector.record(value);
|
|
289
|
+
},
|
|
290
|
+
summarize({ complete }) {
|
|
291
|
+
const summary = summarizeCollected({ ...collector.result(), complete });
|
|
292
|
+
if (signal?.aborted || complete === false) return summary;
|
|
293
|
+
return writeCachedTokens("summary", record.rolloutPath, stamp, summary);
|
|
294
|
+
},
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
export async function readSessionTokens({ codexHome, id, maxLineBytes, signal }) {
|
|
299
|
+
const record = await findSessionRecord({ codexHome, id });
|
|
300
|
+
if (!record) return null;
|
|
301
|
+
if (!record.rolloutPath) return { available: false, reason: "no-transcript-path" };
|
|
302
|
+
|
|
303
|
+
const stamp = await readFileStamp(record.rolloutPath);
|
|
304
|
+
const cached = readCachedTokens("summary", record.rolloutPath, stamp);
|
|
305
|
+
if (cached !== undefined) return cached;
|
|
306
|
+
|
|
307
|
+
const prefix = record.forkedFromId
|
|
308
|
+
? await readParentSignatures({ codexHome, id: record.forkedFromId, maxLineBytes, signal })
|
|
309
|
+
: null;
|
|
310
|
+
|
|
311
|
+
const collected = await collectOrMissing(record.rolloutPath, { maxLineBytes, prefix, signal });
|
|
312
|
+
if (!collected) return { available: false, reason: "transcript-missing" };
|
|
313
|
+
|
|
314
|
+
const summary = summarizeCollected(collected);
|
|
315
|
+
if (signal?.aborted || collected.complete === false) return summary;
|
|
316
|
+
return writeCachedTokens("summary", record.rolloutPath, stamp, summary);
|
|
317
|
+
}
|