sidekick-shared 0.29.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -0
- package/README.md +198 -56
- package/dist/aggregation/EventAggregator.d.ts +96 -2
- package/dist/aggregation/EventAggregator.js +535 -42
- package/dist/aggregation/snapshot.d.ts +37 -8
- package/dist/aggregation/snapshot.js +31 -9
- package/dist/aggregation/types.d.ts +38 -1
- package/dist/browser.d.ts +9 -1
- package/dist/browser.js +29 -3
- package/dist/context/sessionContext.js +8 -0
- package/dist/esm/aggregation/EventAggregator.js +535 -42
- package/dist/esm/aggregation/snapshot.js +31 -9
- package/dist/esm/browser.js +3 -0
- package/dist/esm/context/sessionContext.js +8 -0
- package/dist/esm/fileContinuity.js +149 -0
- package/dist/esm/formatters/toolSummary.js +4 -1
- package/dist/esm/index.js +7 -2
- package/dist/esm/modelCatalog.js +5 -4
- package/dist/esm/modelContext.js +4 -3
- package/dist/esm/modelInfo.js +4 -3
- package/dist/esm/node.js +3 -1
- package/dist/esm/observedContextWindows.js +2 -1
- package/dist/esm/ownKey.js +26 -0
- package/dist/esm/parsers/codexParser.js +372 -7
- package/dist/esm/parsers/openCodeParser.js +2 -1
- package/dist/esm/parsers/subagentEndState.js +61 -0
- package/dist/esm/parsers/subagentScanner.js +37 -12
- package/dist/esm/parsers/subagentTraceParser.js +103 -37
- package/dist/esm/pricingCatalog.js +3 -2
- package/dist/esm/promptHistory.js +22 -11
- package/dist/esm/providers/claudeCode.js +60 -29
- package/dist/esm/providers/codex.js +235 -201
- package/dist/esm/providers/codexCompressedReader.js +184 -0
- package/dist/esm/providers/codexDatabase.js +47 -2
- package/dist/esm/providers/codexRollouts.js +20 -2
- package/dist/esm/providers/codexSubagents.js +456 -0
- package/dist/esm/providers/compressedRollouts.js +330 -0
- package/dist/esm/providers/factory.js +4 -1
- package/dist/esm/providers/rolloutWalker.js +32 -4
- package/dist/esm/schemas/sessionEvent.js +63 -14
- package/dist/esm/sessionMonitor.js +7 -1
- package/dist/esm/sessionPreviews.js +9 -0
- package/dist/esm/sessionPromptHistory.js +5 -3
- package/dist/esm/subagentActivity.js +125 -0
- package/dist/esm/subagentDisplayState.js +81 -0
- package/dist/esm/subagentSpawn.js +363 -0
- package/dist/esm/testing/codexSubagentFixtures.js +336 -0
- package/dist/esm/toolCallOrigins.js +17 -10
- package/dist/esm/toolCategory.js +6 -2
- package/dist/esm/transcript.js +8 -0
- package/dist/esm/turns/assistantTurn.js +6 -2
- package/dist/esm/usage/claudeUsageDedupe.js +68 -0
- package/dist/esm/usage/usageEvents.js +4 -1
- package/dist/esm/watchers/eventBridge.js +45 -0
- package/dist/esm/watchers/factory.js +5 -3
- package/dist/esm/watchers/jsonlTail.js +82 -48
- package/dist/esm/watchers/jsonlWatcher.js +61 -4
- package/dist/esm/watchers/providerReaderWatcher.js +18 -4
- package/dist/esm/zaiQuota.js +2 -1
- package/dist/fileContinuity.d.ts +95 -0
- package/dist/fileContinuity.js +187 -0
- package/dist/formatters/toolSummary.js +4 -1
- package/dist/index.d.ts +19 -7
- package/dist/index.js +48 -10
- package/dist/modelCatalog.js +5 -4
- package/dist/modelContext.js +4 -3
- package/dist/modelInfo.js +4 -3
- package/dist/node.d.ts +5 -1
- package/dist/node.js +18 -2
- package/dist/observedContextWindows.js +2 -1
- package/dist/ownKey.d.ts +16 -0
- package/dist/ownKey.js +30 -0
- package/dist/parsers/codexParser.d.ts +107 -0
- package/dist/parsers/codexParser.js +373 -7
- package/dist/parsers/openCodeParser.js +2 -1
- package/dist/parsers/subagentEndState.d.ts +20 -0
- package/dist/parsers/subagentEndState.js +65 -0
- package/dist/parsers/subagentScanner.d.ts +19 -6
- package/dist/parsers/subagentScanner.js +37 -12
- package/dist/parsers/subagentTraceParser.d.ts +33 -5
- package/dist/parsers/subagentTraceParser.js +103 -37
- package/dist/pricingCatalog.js +3 -2
- package/dist/promptHistory.d.ts +13 -3
- package/dist/promptHistory.js +22 -11
- package/dist/providers/claudeCode.js +59 -28
- package/dist/providers/codex.d.ts +35 -5
- package/dist/providers/codex.js +232 -198
- package/dist/providers/codexCompressedReader.d.ts +58 -0
- package/dist/providers/codexCompressedReader.js +221 -0
- package/dist/providers/codexDatabase.d.ts +25 -0
- package/dist/providers/codexDatabase.js +48 -2
- package/dist/providers/codexRollouts.d.ts +16 -2
- package/dist/providers/codexRollouts.js +20 -2
- package/dist/providers/codexSubagents.d.ts +66 -0
- package/dist/providers/codexSubagents.js +495 -0
- package/dist/providers/compressedRollouts.d.ts +114 -0
- package/dist/providers/compressedRollouts.js +380 -0
- package/dist/providers/factory.d.ts +5 -0
- package/dist/providers/factory.js +4 -1
- package/dist/providers/rolloutWalker.d.ts +15 -2
- package/dist/providers/rolloutWalker.js +33 -4
- package/dist/providers/types.d.ts +36 -1
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/sessionEvent.d.ts +51 -10
- package/dist/schemas/sessionEvent.js +63 -14
- package/dist/sessionMonitor.d.ts +6 -0
- package/dist/sessionMonitor.js +7 -1
- package/dist/sessionPreviews.js +9 -0
- package/dist/sessionPromptHistory.d.ts +5 -0
- package/dist/sessionPromptHistory.js +5 -36
- package/dist/subagentActivity.d.ts +91 -0
- package/dist/subagentActivity.js +132 -0
- package/dist/subagentDisplayState.d.ts +81 -0
- package/dist/subagentDisplayState.js +86 -0
- package/dist/subagentSpawn.d.ts +175 -0
- package/dist/subagentSpawn.js +385 -0
- package/dist/testing/codexSubagentFixtures.d.ts +48 -0
- package/dist/testing/codexSubagentFixtures.js +344 -0
- package/dist/toolCallOrigins.d.ts +5 -0
- package/dist/toolCallOrigins.js +17 -10
- package/dist/toolCategory.js +6 -2
- package/dist/transcript.js +8 -0
- package/dist/turns/assistantTurn.d.ts +4 -0
- package/dist/turns/assistantTurn.js +6 -2
- package/dist/types/codex.d.ts +71 -2
- package/dist/types/sessionEvent.d.ts +38 -3
- package/dist/usage/claudeUsageDedupe.d.ts +31 -0
- package/dist/usage/claudeUsageDedupe.js +68 -0
- package/dist/usage/usageEvents.d.ts +4 -1
- package/dist/usage/usageEvents.js +4 -1
- package/dist/watchers/eventBridge.js +45 -0
- package/dist/watchers/factory.d.ts +6 -0
- package/dist/watchers/factory.js +5 -3
- package/dist/watchers/jsonlTail.d.ts +27 -4
- package/dist/watchers/jsonlTail.js +84 -48
- package/dist/watchers/jsonlWatcher.d.ts +22 -1
- package/dist/watchers/jsonlWatcher.js +60 -3
- package/dist/watchers/providerReaderWatcher.d.ts +7 -0
- package/dist/watchers/providerReaderWatcher.js +18 -4
- package/dist/watchers/types.d.ts +12 -0
- package/dist/zaiQuota.js +2 -1
- package/package.json +1 -1
package/dist/providers/codex.js
CHANGED
|
@@ -5,7 +5,9 @@
|
|
|
5
5
|
* Implements SessionProviderBase for monitoring Codex CLI sessions.
|
|
6
6
|
* Codex stores sessions as JSONL rollout files in:
|
|
7
7
|
* ~/.codex/sessions/YYYY/MM/DD/rollout-<timestamp>-<uuid>.jsonl
|
|
8
|
-
* with an optional SQLite index at ~/.codex/state.sqlite.
|
|
8
|
+
* with an optional SQLite index at ~/.codex/state.sqlite. Older rollouts are
|
|
9
|
+
* compressed in place to `….jsonl.zst`; those are read only with the
|
|
10
|
+
* `includeCompressedRollouts` option (see `providers/compressedRollouts`).
|
|
9
11
|
*
|
|
10
12
|
* Ported from sidekick-vscode/src/services/providers/CodexSessionProvider.ts
|
|
11
13
|
*/
|
|
@@ -51,17 +53,20 @@ const sessionContext_1 = require("../context/sessionContext");
|
|
|
51
53
|
const sessionStats_1 = require("../sessionStats");
|
|
52
54
|
const codexParser_1 = require("../parsers/codexParser");
|
|
53
55
|
const codexDatabase_1 = require("./codexDatabase");
|
|
54
|
-
const subagentStatsCache_1 = require("../parsers/subagentStatsCache");
|
|
55
56
|
const rolloutWalker_1 = require("./rolloutWalker");
|
|
57
|
+
const compressedRollouts_1 = require("./compressedRollouts");
|
|
58
|
+
const codexCompressedReader_1 = require("./codexCompressedReader");
|
|
59
|
+
const codexSubagents_1 = require("./codexSubagents");
|
|
60
|
+
const codexRolloutOrigin_1 = require("../parsers/codexRolloutOrigin");
|
|
56
61
|
const codexPaths_1 = require("../codexPaths");
|
|
57
62
|
const codexRollouts_1 = require("./codexRollouts");
|
|
58
63
|
Object.defineProperty(exports, "findCodexRolloutFile", { enumerable: true, get: function () { return codexRollouts_1.findCodexRolloutFile; } });
|
|
59
64
|
const directoryListingCache_1 = require("./directoryListingCache");
|
|
65
|
+
const fileContinuity_1 = require("../fileContinuity");
|
|
60
66
|
const diagnostics_1 = require("./diagnostics");
|
|
61
67
|
const modelContext_1 = require("../modelContext");
|
|
62
68
|
const observedContextWindows_1 = require("../observedContextWindows");
|
|
63
69
|
const usageNormalization_1 = require("../usageNormalization");
|
|
64
|
-
const subagentUsage_1 = require("../usage/subagentUsage");
|
|
65
70
|
// ---------------------------------------------------------------------------
|
|
66
71
|
// Helper functions
|
|
67
72
|
// ---------------------------------------------------------------------------
|
|
@@ -74,8 +79,8 @@ function truncate(text, maxLen) {
|
|
|
74
79
|
return trimmed;
|
|
75
80
|
}
|
|
76
81
|
/** Every rollout under a directory, newest first (one capped walk). */
|
|
77
|
-
function findRolloutFiles(dir) {
|
|
78
|
-
return (0, rolloutWalker_1.walkRolloutFiles)([dir]);
|
|
82
|
+
function findRolloutFiles(dir, includeCompressed = false) {
|
|
83
|
+
return (0, rolloutWalker_1.walkRolloutFiles)([dir], { includeCompressed });
|
|
79
84
|
}
|
|
80
85
|
function hasRolloutFile(dir) {
|
|
81
86
|
return (0, rolloutWalker_1.walkRolloutFiles)([dir], { limit: 1 }).length > 0;
|
|
@@ -84,6 +89,10 @@ function hasRolloutFile(dir) {
|
|
|
84
89
|
function findRolloutFilesInConfiguredHomes(options = {}) {
|
|
85
90
|
return (0, rolloutWalker_1.walkRolloutFiles)((0, codexRollouts_1.getSessionsDirs)(), options);
|
|
86
91
|
}
|
|
92
|
+
/** Whether a rollout's session_meta says a subagent wrote it. */
|
|
93
|
+
function isSubagentRollout(rolloutPath) {
|
|
94
|
+
return (0, codexRolloutOrigin_1.codexRolloutOrigin)((0, codexRollouts_1.readSessionMeta)(rolloutPath)).isSubagent;
|
|
95
|
+
}
|
|
87
96
|
/** Pick the best sessions directory for UI/search entrypoints. */
|
|
88
97
|
function getSessionsDir() {
|
|
89
98
|
const dirs = (0, codexPaths_1.getCodexSessionsDirs)(undefined, { includeArchived: false });
|
|
@@ -160,19 +169,16 @@ function extractFirstUserMessageFromLines(lines) {
|
|
|
160
169
|
return null;
|
|
161
170
|
}
|
|
162
171
|
async function extractFirstUserMessageAsync(rolloutPath) {
|
|
163
|
-
let handle = null;
|
|
164
172
|
try {
|
|
165
|
-
|
|
166
|
-
const
|
|
167
|
-
|
|
168
|
-
|
|
173
|
+
// A compressed rollout's head is decompressed; a plain one is read as is.
|
|
174
|
+
const head = await (0, compressedRollouts_1.readRolloutHeadAsync)(rolloutPath, 256 * 1024);
|
|
175
|
+
if (head.length === 0)
|
|
176
|
+
return null;
|
|
177
|
+
return extractFirstUserMessageFromLines(head.toString('utf8').split('\n').slice(0, 20));
|
|
169
178
|
}
|
|
170
179
|
catch {
|
|
171
180
|
return null;
|
|
172
181
|
}
|
|
173
|
-
finally {
|
|
174
|
-
await handle?.close().catch(() => undefined);
|
|
175
|
-
}
|
|
176
182
|
}
|
|
177
183
|
/** Extract searchable text from a rollout line. */
|
|
178
184
|
function extractSearchableText(line) {
|
|
@@ -304,50 +310,62 @@ class CodexReader {
|
|
|
304
310
|
/** Cursor after the last complete line, safe to persist across restarts. */
|
|
305
311
|
committedPosition = 0;
|
|
306
312
|
lineBuffer = '';
|
|
307
|
-
|
|
313
|
+
/** Why the last `readNew()` started over from offset 0, if it did. */
|
|
314
|
+
_lastReset = null;
|
|
315
|
+
/**
|
|
316
|
+
* A restart found while settling a seek (the file is now shorter than the
|
|
317
|
+
* position sought); the next `readNew()` reports it.
|
|
318
|
+
*/
|
|
319
|
+
pendingRestart = null;
|
|
320
|
+
/** Whether the rollout is still the one the positions refer to. */
|
|
321
|
+
continuity = new fileContinuity_1.FileContinuity();
|
|
308
322
|
decoder = new string_decoder_1.StringDecoder('utf8');
|
|
323
|
+
/**
|
|
324
|
+
* Position from `seekTo()` whose prefix still has to be parsed for model,
|
|
325
|
+
* turn, and cumulative-token context; `restoreResumeState()` supplies that
|
|
326
|
+
* context instead and clears it.
|
|
327
|
+
*/
|
|
328
|
+
pendingSeek = null;
|
|
309
329
|
constructor(rolloutPath, onContextWindowLimit, onRateLimits, onTokenUsage) {
|
|
310
330
|
this.rolloutPath = rolloutPath;
|
|
311
331
|
this.onContextWindowLimit = onContextWindowLimit;
|
|
312
332
|
this.onRateLimits = onRateLimits;
|
|
313
333
|
this.onTokenUsage = onTokenUsage;
|
|
314
|
-
this.parser = new codexParser_1.CodexRolloutParser();
|
|
334
|
+
this.parser = new codexParser_1.CodexRolloutParser({ includeSubagentActivity: true });
|
|
315
335
|
}
|
|
316
336
|
/** Get the parser's session metadata. */
|
|
317
337
|
getSessionMeta() {
|
|
338
|
+
this.settleSeek();
|
|
318
339
|
return this.parser.getSessionMeta();
|
|
319
340
|
}
|
|
320
341
|
readNew(endPosition) {
|
|
321
|
-
this.
|
|
342
|
+
this.settleSeek();
|
|
343
|
+
this._lastReset = null;
|
|
322
344
|
const events = [];
|
|
345
|
+
let fd = null;
|
|
323
346
|
try {
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
const
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
this.
|
|
331
|
-
this.filePosition = 0;
|
|
332
|
-
this.committedPosition = 0;
|
|
333
|
-
this.lineBuffer = '';
|
|
334
|
-
this.parser.reset();
|
|
335
|
-
this.decoder = new string_decoder_1.StringDecoder('utf8');
|
|
347
|
+
// Open first and stat the descriptor: the size and identity checked
|
|
348
|
+
// are those of the file actually read, even if the path is replaced.
|
|
349
|
+
fd = fs.openSync(this.rolloutPath, 'r');
|
|
350
|
+
const stats = fs.fstatSync(fd);
|
|
351
|
+
if (this.pendingRestart) {
|
|
352
|
+
this._lastReset = this.pendingRestart;
|
|
353
|
+
this.pendingRestart = null;
|
|
336
354
|
}
|
|
355
|
+
const reason = this.continuity.check(fd, stats, this.filePosition, this.committedPosition);
|
|
356
|
+
// Truncated, rewritten, or replaced: what was read no longer describes
|
|
357
|
+
// the rollout, so start over from its beginning.
|
|
358
|
+
if (reason !== null)
|
|
359
|
+
this.restart(reason);
|
|
360
|
+
this.continuity.adopt(stats);
|
|
361
|
+
const currentSize = Math.min(stats.size, endPosition ?? stats.size);
|
|
337
362
|
// No new content
|
|
338
363
|
if (currentSize <= this.filePosition)
|
|
339
|
-
return
|
|
364
|
+
return events;
|
|
340
365
|
// Read new bytes
|
|
341
|
-
const fd = fs.openSync(this.rolloutPath, 'r');
|
|
342
366
|
const bufferSize = currentSize - this.filePosition;
|
|
343
367
|
const buffer = Buffer.alloc(bufferSize);
|
|
344
|
-
|
|
345
|
-
try {
|
|
346
|
-
bytesRead = fs.readSync(fd, buffer, 0, bufferSize, this.filePosition);
|
|
347
|
-
}
|
|
348
|
-
finally {
|
|
349
|
-
fs.closeSync(fd);
|
|
350
|
-
}
|
|
368
|
+
const bytesRead = fs.readSync(fd, buffer, 0, bufferSize, this.filePosition);
|
|
351
369
|
const bytes = buffer.subarray(0, bytesRead);
|
|
352
370
|
const newline = bytes.lastIndexOf(0x0a);
|
|
353
371
|
if (newline >= 0)
|
|
@@ -371,47 +389,71 @@ class CodexReader {
|
|
|
371
389
|
// Skip malformed JSON lines
|
|
372
390
|
}
|
|
373
391
|
}
|
|
374
|
-
|
|
375
|
-
// The model comes along so the value can be persisted per model — the
|
|
376
|
-
// window Codex reports is tier-specific, not the published maximum.
|
|
377
|
-
const mcw = this.parser.getModelContextWindow();
|
|
378
|
-
if (mcw && this.onContextWindowLimit) {
|
|
379
|
-
this.onContextWindowLimit(mcw, this.parser.getCurrentModel() ?? undefined);
|
|
380
|
-
}
|
|
381
|
-
// Propagate rate_limits from token_count events to the provider
|
|
382
|
-
const rateLimits = this.parser.getLastRateLimits();
|
|
383
|
-
if (rateLimits && this.onRateLimits) {
|
|
384
|
-
this.onRateLimits(rateLimits);
|
|
385
|
-
}
|
|
386
|
-
// Propagate last token usage snapshot to the provider
|
|
387
|
-
const lastUsage = this.parser.getLastTokenUsage();
|
|
388
|
-
if (lastUsage && this.onTokenUsage) {
|
|
389
|
-
const normalized = (0, usageNormalization_1.normalizeProviderUsage)({
|
|
390
|
-
semantics: 'openai',
|
|
391
|
-
provider: 'openai',
|
|
392
|
-
source: 'codex-reader-snapshot',
|
|
393
|
-
model: this.parser.getCurrentModel() || undefined,
|
|
394
|
-
inputTokens: lastUsage.input_tokens,
|
|
395
|
-
outputTokens: lastUsage.output_tokens,
|
|
396
|
-
cacheReadTokens: lastUsage.cached_input_tokens,
|
|
397
|
-
reasoningTokens: lastUsage.reasoning_output_tokens,
|
|
398
|
-
});
|
|
399
|
-
this.onTokenUsage({
|
|
400
|
-
inputTokens: normalized.uncachedInputTokens,
|
|
401
|
-
outputTokens: normalized.outputTokens,
|
|
402
|
-
cacheWriteTokens: normalized.cacheWriteTokens,
|
|
403
|
-
cacheReadTokens: normalized.cacheReadTokens,
|
|
404
|
-
reasoningTokens: normalized.reasoningTokens,
|
|
405
|
-
model: this.parser.getCurrentModel() || 'unknown',
|
|
406
|
-
timestamp: new Date(),
|
|
407
|
-
});
|
|
408
|
-
}
|
|
392
|
+
this.propagateParserState();
|
|
409
393
|
}
|
|
410
394
|
catch {
|
|
411
|
-
// Skip read errors
|
|
395
|
+
// Skip read errors (a missing rollout reads as no events)
|
|
396
|
+
}
|
|
397
|
+
finally {
|
|
398
|
+
if (fd !== null) {
|
|
399
|
+
try {
|
|
400
|
+
fs.closeSync(fd);
|
|
401
|
+
}
|
|
402
|
+
catch {
|
|
403
|
+
// Ignore close errors.
|
|
404
|
+
}
|
|
405
|
+
}
|
|
412
406
|
}
|
|
413
407
|
return events;
|
|
414
408
|
}
|
|
409
|
+
/** Starts over from offset 0 of a rollout that was truncated, rewritten, or replaced. */
|
|
410
|
+
restart(reason) {
|
|
411
|
+
this._lastReset = { reason, previousPosition: this.committedPosition };
|
|
412
|
+
this.filePosition = 0;
|
|
413
|
+
this.committedPosition = 0;
|
|
414
|
+
this.lineBuffer = '';
|
|
415
|
+
this.parser.reset();
|
|
416
|
+
this.decoder = new string_decoder_1.StringDecoder('utf8');
|
|
417
|
+
this.pendingSeek = null;
|
|
418
|
+
}
|
|
419
|
+
/** Hands the parser's context window, rate limits, and last usage to the provider. */
|
|
420
|
+
propagateParserState() {
|
|
421
|
+
// Propagate model_context_window from token_count events to the provider.
|
|
422
|
+
// The model comes along so the value can be persisted per model — the
|
|
423
|
+
// window Codex reports is tier-specific, not the published maximum.
|
|
424
|
+
const mcw = this.parser.getModelContextWindow();
|
|
425
|
+
if (mcw && this.onContextWindowLimit) {
|
|
426
|
+
this.onContextWindowLimit(mcw, this.parser.getCurrentModel() ?? undefined);
|
|
427
|
+
}
|
|
428
|
+
// Propagate rate_limits from token_count events to the provider
|
|
429
|
+
const rateLimits = this.parser.getLastRateLimits();
|
|
430
|
+
if (rateLimits && this.onRateLimits) {
|
|
431
|
+
this.onRateLimits(rateLimits);
|
|
432
|
+
}
|
|
433
|
+
// Propagate last token usage snapshot to the provider
|
|
434
|
+
const lastUsage = this.parser.getLastTokenUsage();
|
|
435
|
+
if (lastUsage && this.onTokenUsage) {
|
|
436
|
+
const normalized = (0, usageNormalization_1.normalizeProviderUsage)({
|
|
437
|
+
semantics: 'openai',
|
|
438
|
+
provider: 'openai',
|
|
439
|
+
source: 'codex-reader-snapshot',
|
|
440
|
+
model: this.parser.getCurrentModel() || undefined,
|
|
441
|
+
inputTokens: lastUsage.input_tokens,
|
|
442
|
+
outputTokens: lastUsage.output_tokens,
|
|
443
|
+
cacheReadTokens: lastUsage.cached_input_tokens,
|
|
444
|
+
reasoningTokens: lastUsage.reasoning_output_tokens,
|
|
445
|
+
});
|
|
446
|
+
this.onTokenUsage({
|
|
447
|
+
inputTokens: normalized.uncachedInputTokens,
|
|
448
|
+
outputTokens: normalized.outputTokens,
|
|
449
|
+
cacheWriteTokens: normalized.cacheWriteTokens,
|
|
450
|
+
cacheReadTokens: normalized.cacheReadTokens,
|
|
451
|
+
reasoningTokens: normalized.reasoningTokens,
|
|
452
|
+
model: this.parser.getCurrentModel() || 'unknown',
|
|
453
|
+
timestamp: new Date(),
|
|
454
|
+
});
|
|
455
|
+
}
|
|
456
|
+
}
|
|
415
457
|
readAll() {
|
|
416
458
|
this.reset();
|
|
417
459
|
return this.readNew();
|
|
@@ -422,12 +464,17 @@ class CodexReader {
|
|
|
422
464
|
this.lineBuffer = '';
|
|
423
465
|
this.parser.reset();
|
|
424
466
|
this.decoder = new string_decoder_1.StringDecoder('utf8');
|
|
425
|
-
this.
|
|
467
|
+
this._lastReset = null;
|
|
468
|
+
this.pendingRestart = null;
|
|
469
|
+
this.pendingSeek = null;
|
|
470
|
+
// The next read adopts whatever rollout has the path.
|
|
471
|
+
this.continuity.forget();
|
|
426
472
|
}
|
|
427
473
|
exists() {
|
|
428
474
|
return fs.existsSync(this.rolloutPath);
|
|
429
475
|
}
|
|
430
476
|
flush() {
|
|
477
|
+
this.settleSeek();
|
|
431
478
|
this.lineBuffer += this.decoder.end();
|
|
432
479
|
// Process any remaining data in the line buffer
|
|
433
480
|
if (this.lineBuffer.trim()) {
|
|
@@ -445,19 +492,55 @@ class CodexReader {
|
|
|
445
492
|
return this.committedPosition;
|
|
446
493
|
}
|
|
447
494
|
seekTo(position) {
|
|
448
|
-
// Rebuild model, turn, and cumulative-token context before resuming.
|
|
449
|
-
// The prefix is parsed only; its events are already in the consumer snapshot.
|
|
450
495
|
this.reset();
|
|
496
|
+
this.filePosition = position;
|
|
497
|
+
this.committedPosition = position;
|
|
498
|
+
// The prefix's context is rebuilt on the next read, unless
|
|
499
|
+
// restoreResumeState() supplies it first.
|
|
451
500
|
if (position > 0)
|
|
452
|
-
this.
|
|
501
|
+
this.pendingSeek = position;
|
|
453
502
|
}
|
|
454
503
|
wasTruncated() {
|
|
455
|
-
return this.
|
|
504
|
+
return this._lastReset !== null;
|
|
505
|
+
}
|
|
506
|
+
lastReset() {
|
|
507
|
+
return this._lastReset ? { ...this._lastReset } : null;
|
|
508
|
+
}
|
|
509
|
+
/** The parser's continuation state (see `CodexRolloutParser.exportResumeState`). */
|
|
510
|
+
getResumeState() {
|
|
511
|
+
this.settleSeek();
|
|
512
|
+
return this.parser.exportResumeState();
|
|
513
|
+
}
|
|
514
|
+
/**
|
|
515
|
+
* Restores parser state from `getResumeState()`; after `seekTo()` this
|
|
516
|
+
* replaces re-parsing the rollout up to that position.
|
|
517
|
+
*/
|
|
518
|
+
restoreResumeState(state) {
|
|
519
|
+
if (!this.parser.importResumeState(state))
|
|
520
|
+
return false;
|
|
521
|
+
this.pendingSeek = null;
|
|
522
|
+
this.propagateParserState();
|
|
523
|
+
return true;
|
|
524
|
+
}
|
|
525
|
+
/**
|
|
526
|
+
* Rebuilds model, turn, and cumulative-token context for a pending
|
|
527
|
+
* `seekTo()` position. The prefix is parsed only; its events are already
|
|
528
|
+
* in the consumer snapshot.
|
|
529
|
+
*/
|
|
530
|
+
settleSeek() {
|
|
531
|
+
const position = this.pendingSeek;
|
|
532
|
+
if (position === null)
|
|
533
|
+
return;
|
|
534
|
+
this.reset();
|
|
535
|
+
this.readNew(position);
|
|
536
|
+
if (this.filePosition < position) {
|
|
537
|
+
// The rollout no longer reaches the position sought: it was truncated
|
|
538
|
+
// or replaced since. The next read starts over and reports it.
|
|
539
|
+
this.reset();
|
|
540
|
+
this.pendingRestart = { reason: 'truncated', previousPosition: position };
|
|
541
|
+
}
|
|
456
542
|
}
|
|
457
543
|
}
|
|
458
|
-
// ---------------------------------------------------------------------------
|
|
459
|
-
// CodexProvider — full SessionProviderBase implementation
|
|
460
|
-
// ---------------------------------------------------------------------------
|
|
461
544
|
/**
|
|
462
545
|
* Session provider for Codex CLI.
|
|
463
546
|
*
|
|
@@ -476,8 +559,26 @@ class CodexProvider {
|
|
|
476
559
|
lastTokenUsageData = null;
|
|
477
560
|
/** Dated rollout-directory listings reused across async enumerations. */
|
|
478
561
|
directoryCache = new directoryListingCache_1.DirectoryListingCache();
|
|
562
|
+
/** See {@link CodexProviderOptions.includeCompressedRollouts}. */
|
|
563
|
+
compressedRequested;
|
|
564
|
+
onDiagnostic;
|
|
479
565
|
constructor(options = {}) {
|
|
480
566
|
this.diagnostics = new diagnostics_1.ProviderDiagnosticTracker(this.id, options);
|
|
567
|
+
this.compressedRequested = options.includeCompressedRollouts === true;
|
|
568
|
+
this.onDiagnostic = options.onDiagnostic;
|
|
569
|
+
}
|
|
570
|
+
/**
|
|
571
|
+
* Whether compressed rollouts are discovered and read: requested, and the
|
|
572
|
+
* runtime can decompress them (otherwise reported once, on first use).
|
|
573
|
+
*/
|
|
574
|
+
get includeCompressed() {
|
|
575
|
+
return (this.compressedRequested && (0, compressedRollouts_1.canReadCompressedRollouts)({ onDiagnostic: this.onDiagnostic }));
|
|
576
|
+
}
|
|
577
|
+
/** A stored rollout path as it exists now (its `.zst` when compressed and enabled). */
|
|
578
|
+
resolveRollout(rolloutPath) {
|
|
579
|
+
return rolloutPath
|
|
580
|
+
? (0, compressedRollouts_1.resolveRolloutPath)(rolloutPath, { includeCompressed: this.includeCompressed })
|
|
581
|
+
: null;
|
|
481
582
|
}
|
|
482
583
|
// --- Database ---
|
|
483
584
|
/** Lazy-initialize database connection. */
|
|
@@ -575,10 +676,11 @@ class CodexProvider {
|
|
|
575
676
|
return thread.rollout_path;
|
|
576
677
|
}
|
|
577
678
|
}
|
|
578
|
-
// File scan fallback (newest first; cwd reads are cached per path)
|
|
679
|
+
// File scan fallback (newest first; cwd reads are cached per path).
|
|
680
|
+
// Subagent rollouts share their parent's cwd but are not the session.
|
|
579
681
|
for (const file of findRolloutFilesInConfiguredHomes()) {
|
|
580
682
|
const cwd = (0, codexRollouts_1.readSessionCwd)(file.path);
|
|
581
|
-
if (cwd && (0, codexRollouts_1.cwdMatches)(cwd, workspacePath)) {
|
|
683
|
+
if (cwd && (0, codexRollouts_1.cwdMatches)(cwd, workspacePath) && !isSubagentRollout(file.path)) {
|
|
582
684
|
this.recordDatabaseFallback('findActiveSession', true);
|
|
583
685
|
return file.path;
|
|
584
686
|
}
|
|
@@ -592,15 +694,17 @@ class CodexProvider {
|
|
|
592
694
|
if (db) {
|
|
593
695
|
const threads = db.getThreadsByCwd(workspacePath);
|
|
594
696
|
const dbPaths = threads
|
|
595
|
-
.
|
|
596
|
-
.
|
|
697
|
+
.map((t) => this.resolveRollout(t.rollout_path))
|
|
698
|
+
.filter((rolloutPath) => rolloutPath !== null);
|
|
597
699
|
if (dbPaths.length > 0) {
|
|
598
700
|
this.recordDatabaseFallback('findAllSessions', true);
|
|
599
701
|
return dbPaths;
|
|
600
702
|
}
|
|
601
703
|
}
|
|
602
704
|
// File scan (newest first; cwd reads are cached per path)
|
|
603
|
-
const sessions = findRolloutFilesInConfiguredHomes(
|
|
705
|
+
const sessions = findRolloutFilesInConfiguredHomes({
|
|
706
|
+
includeCompressed: this.includeCompressed,
|
|
707
|
+
})
|
|
604
708
|
.filter((f) => {
|
|
605
709
|
const cwd = (0, codexRollouts_1.readSessionCwd)(f.path);
|
|
606
710
|
return cwd !== null && (0, codexRollouts_1.cwdMatches)(cwd, workspacePath);
|
|
@@ -611,7 +715,7 @@ class CodexProvider {
|
|
|
611
715
|
}
|
|
612
716
|
findSessionById(_workspacePath, sessionId) {
|
|
613
717
|
try {
|
|
614
|
-
const found = (0, codexRollouts_1.findCodexRolloutFile)(sessionId);
|
|
718
|
+
const found = (0, codexRollouts_1.findCodexRolloutFile)(sessionId, { includeCompressed: this.includeCompressed });
|
|
615
719
|
this.recordDatabaseFallback('findSessionById', true);
|
|
616
720
|
return found;
|
|
617
721
|
}
|
|
@@ -625,15 +729,15 @@ class CodexProvider {
|
|
|
625
729
|
return this.findAllSessions(workspacePath);
|
|
626
730
|
}
|
|
627
731
|
findSessionsInDirectory(dir) {
|
|
628
|
-
return findRolloutFiles(dir).map((f) => f.path);
|
|
732
|
+
return findRolloutFiles(dir, this.includeCompressed).map((f) => f.path);
|
|
629
733
|
}
|
|
630
734
|
listAllSessionFiles() {
|
|
631
|
-
const files = findRolloutFilesInConfiguredHomes();
|
|
735
|
+
const files = findRolloutFilesInConfiguredHomes({ includeCompressed: this.includeCompressed });
|
|
632
736
|
this.recordDatabaseFallback('listAllSessionFiles', true);
|
|
633
737
|
return files;
|
|
634
738
|
}
|
|
635
739
|
async listSessionFilesAsync(workspacePath, options = {}) {
|
|
636
|
-
const results = await (0, codexRollouts_1.listCodexSessionFilesAsync)(workspacePath, options, this.directoryCache);
|
|
740
|
+
const results = await (0, codexRollouts_1.listCodexSessionFilesAsync)(workspacePath, { ...options, includeCompressed: this.includeCompressed }, this.directoryCache);
|
|
637
741
|
// Filesystem enumeration bypasses the database, so record that here as
|
|
638
742
|
// the sync listing paths do; otherwise `getLastOperationStatus()` keeps
|
|
639
743
|
// describing whichever operation ran before this one.
|
|
@@ -653,7 +757,7 @@ class CodexProvider {
|
|
|
653
757
|
const name = segments[segments.length - 1];
|
|
654
758
|
if (segments.length - 1 > rolloutWalker_1.DEFAULT_ROLLOUT_WALK_MAX_DEPTH)
|
|
655
759
|
return { status: 'ignored' };
|
|
656
|
-
if (!
|
|
760
|
+
if (!this.isSessionFile(name)) {
|
|
657
761
|
this.directoryCache.invalidate(fullPath);
|
|
658
762
|
try {
|
|
659
763
|
const stat = await fs.promises.stat(fullPath);
|
|
@@ -703,7 +807,9 @@ class CodexProvider {
|
|
|
703
807
|
// after its last update can be missing from it, so older files are not
|
|
704
808
|
// opened; the cwd read itself is cached per path either way.
|
|
705
809
|
const indexedUpTo = db ? db.getDbMtime() : 0;
|
|
706
|
-
for (const file of findRolloutFilesInConfiguredHomes(
|
|
810
|
+
for (const file of findRolloutFilesInConfiguredHomes({
|
|
811
|
+
includeCompressed: this.includeCompressed,
|
|
812
|
+
})) {
|
|
707
813
|
if (db && indexedUpTo > 0 && file.mtime.getTime() <= indexedUpTo)
|
|
708
814
|
continue;
|
|
709
815
|
const cwd = (0, codexRollouts_1.readSessionCwd)(file.path);
|
|
@@ -745,7 +851,7 @@ class CodexProvider {
|
|
|
745
851
|
}
|
|
746
852
|
// --- File identification ---
|
|
747
853
|
isSessionFile(filename) {
|
|
748
|
-
return (0, rolloutWalker_1.isRolloutFile)(filename);
|
|
854
|
+
return (0, rolloutWalker_1.isRolloutFile)(filename) || (this.includeCompressed && (0, rolloutWalker_1.isCompressedRolloutFile)(filename));
|
|
749
855
|
}
|
|
750
856
|
getSessionId(sessionPath) {
|
|
751
857
|
return (0, rolloutWalker_1.extractRolloutSessionId)(path.basename(sessionPath));
|
|
@@ -806,6 +912,17 @@ class CodexProvider {
|
|
|
806
912
|
}
|
|
807
913
|
// --- Data reading ---
|
|
808
914
|
createReader(sessionPath) {
|
|
915
|
+
if ((0, compressedRollouts_1.isCompressedRolloutPath)(sessionPath)) {
|
|
916
|
+
// Compressed rollouts are complete: read whole, never tailed.
|
|
917
|
+
return new codexCompressedReader_1.CompressedCodexRolloutReader(sessionPath, {
|
|
918
|
+
onContextWindowLimit: (limit) => {
|
|
919
|
+
// Historical content: keep the live limit, do not persist it again.
|
|
920
|
+
if (this.dynamicContextWindowLimit === null)
|
|
921
|
+
this.dynamicContextWindowLimit = limit;
|
|
922
|
+
},
|
|
923
|
+
onDiagnostic: this.onDiagnostic,
|
|
924
|
+
});
|
|
925
|
+
}
|
|
809
926
|
return new CodexReader(sessionPath, (limit, modelId) => {
|
|
810
927
|
this.dynamicContextWindowLimit = limit;
|
|
811
928
|
if (modelId) {
|
|
@@ -822,119 +939,36 @@ class CodexProvider {
|
|
|
822
939
|
});
|
|
823
940
|
}
|
|
824
941
|
// --- Subagent support ---
|
|
825
|
-
scanSubagents(_sessionDir, sessionId) {
|
|
826
|
-
// Codex tracks forked sessions via forked_from_id in session_meta.
|
|
827
|
-
// Use DB when available, fall back to filesystem scanning.
|
|
828
|
-
const subagents = [];
|
|
829
|
-
// DB approach: spawned subagents (parent recorded in `threads.source`),
|
|
830
|
-
// plus forked children on older schemas that carry `forked_from_id`.
|
|
831
|
-
const db = this.ensureDb();
|
|
832
|
-
if (db) {
|
|
833
|
-
const seen = new Set();
|
|
834
|
-
const children = [
|
|
835
|
-
...db.getSpawnedChildThreads(sessionId),
|
|
836
|
-
...db.getThreadsByForkedFromId(sessionId),
|
|
837
|
-
].filter((thread) => !seen.has(thread.id) && seen.add(thread.id));
|
|
838
|
-
for (const thread of children) {
|
|
839
|
-
const built = (0, subagentStatsCache_1.cachedSubagentStats)(thread.rollout_path, () => this.buildSubagentStats(thread.id, thread.rollout_path, thread.title || thread.agent_nickname));
|
|
840
|
-
if (built) {
|
|
841
|
-
subagents.push(thread.agent_role && !built.agentType
|
|
842
|
-
? { ...built, agentType: thread.agent_role }
|
|
843
|
-
: built);
|
|
844
|
-
}
|
|
845
|
-
}
|
|
846
|
-
if (children.length > 0)
|
|
847
|
-
return subagents;
|
|
848
|
-
}
|
|
849
|
-
// Filesystem fallback: scan recent rollout files for a matching
|
|
850
|
-
// forked_from_id. The walk stops after the 50 newest-dated files instead
|
|
851
|
-
// of enumerating the whole history and slicing afterwards.
|
|
852
|
-
for (const file of findRolloutFilesInConfiguredHomes({ limit: 50 })) {
|
|
853
|
-
const meta = (0, codexRollouts_1.readSessionMeta)(file.path);
|
|
854
|
-
if (meta &&
|
|
855
|
-
(meta.forked_from_id === sessionId || (0, codexDatabase_1.spawnParentFromSource)(meta.source) === sessionId)) {
|
|
856
|
-
const childId = (0, rolloutWalker_1.extractRolloutSessionId)(path.basename(file.path));
|
|
857
|
-
// `source` is a plain string for top-level sessions but an object
|
|
858
|
-
// (`{ subagent: { thread_spawn: … } }`) for spawned children.
|
|
859
|
-
const description = typeof meta.source === 'string' ? meta.source : undefined;
|
|
860
|
-
const stats = (0, subagentStatsCache_1.cachedSubagentStats)(file.path, () => this.buildSubagentStats(childId, file.path, description));
|
|
861
|
-
if (stats)
|
|
862
|
-
subagents.push(stats);
|
|
863
|
-
}
|
|
864
|
-
}
|
|
865
|
-
return subagents;
|
|
866
|
-
}
|
|
867
942
|
/**
|
|
868
|
-
*
|
|
943
|
+
* Every subagent below the session, any depth: spawned threads (the
|
|
944
|
+
* database's `threads.source`, else the newest rollouts' session_meta)
|
|
945
|
+
* and, on older schemas, forks (`forked_from_id`). Each carries its
|
|
946
|
+
* rollout (`filePath`), the parent's spawn call (`spawnToolUseId`),
|
|
947
|
+
* `agentNickname`, `agentPath`, `parentAgentId`, and `depth`; tool calls
|
|
948
|
+
* and tokens come from the child's own records, not the parent history a
|
|
949
|
+
* forked child embeds.
|
|
869
950
|
*/
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
const content = fs.readFileSync(rolloutPath, 'utf8');
|
|
883
|
-
const lines = content.split('\n');
|
|
884
|
-
for (const line of lines) {
|
|
885
|
-
const trimmed = line.trim();
|
|
886
|
-
if (!trimmed)
|
|
887
|
-
continue;
|
|
888
|
-
try {
|
|
889
|
-
const parsed = JSON.parse(trimmed);
|
|
890
|
-
// Track timestamps
|
|
891
|
-
if (parsed.timestamp) {
|
|
892
|
-
const ts = new Date(parsed.timestamp);
|
|
893
|
-
if (!stats.startTime || ts < stats.startTime)
|
|
894
|
-
stats.startTime = ts;
|
|
895
|
-
if (!stats.endTime || ts > stats.endTime)
|
|
896
|
-
stats.endTime = ts;
|
|
897
|
-
}
|
|
898
|
-
const events = parser.convertLine(parsed);
|
|
899
|
-
for (const event of events) {
|
|
900
|
-
// Collect tool calls
|
|
901
|
-
if (event.type === 'assistant' && Array.isArray(event.message.content)) {
|
|
902
|
-
for (const block of event.message.content) {
|
|
903
|
-
if (block.type === 'tool_use' && typeof block.name === 'string') {
|
|
904
|
-
stats.toolCalls.push({
|
|
905
|
-
name: block.name,
|
|
906
|
-
input: block.input || {},
|
|
907
|
-
timestamp: new Date(event.timestamp),
|
|
908
|
-
});
|
|
909
|
-
}
|
|
910
|
-
}
|
|
911
|
-
}
|
|
912
|
-
// Accumulate tokens from usage events (duplicate token_count events
|
|
913
|
-
// are already dropped by the parser).
|
|
914
|
-
const usage = (0, usageNormalization_1.extractNormalizedUsage)(event);
|
|
915
|
-
if (usage)
|
|
916
|
-
(0, subagentUsage_1.addUsageToSubagent)(stats, usage);
|
|
917
|
-
}
|
|
918
|
-
}
|
|
919
|
-
catch {
|
|
920
|
-
/* skip malformed lines */
|
|
921
|
-
}
|
|
922
|
-
}
|
|
923
|
-
if (stats.startTime && stats.endTime) {
|
|
924
|
-
stats.durationMs = stats.endTime.getTime() - stats.startTime.getTime();
|
|
925
|
-
}
|
|
926
|
-
}
|
|
927
|
-
catch {
|
|
928
|
-
/* skip unreadable files */
|
|
929
|
-
}
|
|
930
|
-
return stats;
|
|
951
|
+
scanSubagents(_sessionDir, sessionId) {
|
|
952
|
+
const db = this.ensureDb();
|
|
953
|
+
return (0, codexSubagents_1.discoverCodexSubagents)(sessionId, {
|
|
954
|
+
db,
|
|
955
|
+
includeCompressed: this.includeCompressed,
|
|
956
|
+
onDiagnostic: this.onDiagnostic,
|
|
957
|
+
sessionRolloutPath: () => this.resolveRollout(db?.getThread(sessionId)?.rollout_path) ??
|
|
958
|
+
(0, codexRollouts_1.findCodexRolloutFile)(sessionId, { includeCompressed: this.includeCompressed }),
|
|
959
|
+
// The walk stops after the 50 newest-dated files instead of
|
|
960
|
+
// enumerating the whole history and slicing afterwards.
|
|
961
|
+
fallbackFiles: () => findRolloutFilesInConfiguredHomes({ limit: 50, includeCompressed: this.includeCompressed }),
|
|
962
|
+
});
|
|
931
963
|
}
|
|
932
964
|
// --- Cross-session search ---
|
|
933
965
|
searchInSession(sessionPath, query, maxResults) {
|
|
934
966
|
const results = [];
|
|
935
967
|
const queryLower = query.toLowerCase();
|
|
936
968
|
try {
|
|
937
|
-
const content =
|
|
969
|
+
const content = (0, compressedRollouts_1.readRolloutText)(sessionPath, { onDiagnostic: this.onDiagnostic });
|
|
970
|
+
if (content === null)
|
|
971
|
+
return results;
|
|
938
972
|
const lines = content.split('\n');
|
|
939
973
|
for (const line of lines) {
|
|
940
974
|
if (results.length >= maxResults)
|