sidekick-shared 0.29.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -0
- package/README.md +198 -56
- package/dist/aggregation/EventAggregator.d.ts +96 -2
- package/dist/aggregation/EventAggregator.js +535 -42
- package/dist/aggregation/snapshot.d.ts +37 -8
- package/dist/aggregation/snapshot.js +31 -9
- package/dist/aggregation/types.d.ts +38 -1
- package/dist/browser.d.ts +9 -1
- package/dist/browser.js +29 -3
- package/dist/context/sessionContext.js +8 -0
- package/dist/esm/aggregation/EventAggregator.js +535 -42
- package/dist/esm/aggregation/snapshot.js +31 -9
- package/dist/esm/browser.js +3 -0
- package/dist/esm/context/sessionContext.js +8 -0
- package/dist/esm/fileContinuity.js +149 -0
- package/dist/esm/formatters/toolSummary.js +4 -1
- package/dist/esm/index.js +7 -2
- package/dist/esm/modelCatalog.js +5 -4
- package/dist/esm/modelContext.js +4 -3
- package/dist/esm/modelInfo.js +4 -3
- package/dist/esm/node.js +3 -1
- package/dist/esm/observedContextWindows.js +2 -1
- package/dist/esm/ownKey.js +26 -0
- package/dist/esm/parsers/codexParser.js +372 -7
- package/dist/esm/parsers/openCodeParser.js +2 -1
- package/dist/esm/parsers/subagentEndState.js +61 -0
- package/dist/esm/parsers/subagentScanner.js +37 -12
- package/dist/esm/parsers/subagentTraceParser.js +103 -37
- package/dist/esm/pricingCatalog.js +3 -2
- package/dist/esm/promptHistory.js +22 -11
- package/dist/esm/providers/claudeCode.js +60 -29
- package/dist/esm/providers/codex.js +235 -201
- package/dist/esm/providers/codexCompressedReader.js +184 -0
- package/dist/esm/providers/codexDatabase.js +47 -2
- package/dist/esm/providers/codexRollouts.js +20 -2
- package/dist/esm/providers/codexSubagents.js +456 -0
- package/dist/esm/providers/compressedRollouts.js +330 -0
- package/dist/esm/providers/factory.js +4 -1
- package/dist/esm/providers/rolloutWalker.js +32 -4
- package/dist/esm/schemas/sessionEvent.js +63 -14
- package/dist/esm/sessionMonitor.js +7 -1
- package/dist/esm/sessionPreviews.js +9 -0
- package/dist/esm/sessionPromptHistory.js +5 -3
- package/dist/esm/subagentActivity.js +125 -0
- package/dist/esm/subagentDisplayState.js +81 -0
- package/dist/esm/subagentSpawn.js +363 -0
- package/dist/esm/testing/codexSubagentFixtures.js +336 -0
- package/dist/esm/toolCallOrigins.js +17 -10
- package/dist/esm/toolCategory.js +6 -2
- package/dist/esm/transcript.js +8 -0
- package/dist/esm/turns/assistantTurn.js +6 -2
- package/dist/esm/usage/claudeUsageDedupe.js +68 -0
- package/dist/esm/usage/usageEvents.js +4 -1
- package/dist/esm/watchers/eventBridge.js +45 -0
- package/dist/esm/watchers/factory.js +5 -3
- package/dist/esm/watchers/jsonlTail.js +82 -48
- package/dist/esm/watchers/jsonlWatcher.js +61 -4
- package/dist/esm/watchers/providerReaderWatcher.js +18 -4
- package/dist/esm/zaiQuota.js +2 -1
- package/dist/fileContinuity.d.ts +95 -0
- package/dist/fileContinuity.js +187 -0
- package/dist/formatters/toolSummary.js +4 -1
- package/dist/index.d.ts +19 -7
- package/dist/index.js +48 -10
- package/dist/modelCatalog.js +5 -4
- package/dist/modelContext.js +4 -3
- package/dist/modelInfo.js +4 -3
- package/dist/node.d.ts +5 -1
- package/dist/node.js +18 -2
- package/dist/observedContextWindows.js +2 -1
- package/dist/ownKey.d.ts +16 -0
- package/dist/ownKey.js +30 -0
- package/dist/parsers/codexParser.d.ts +107 -0
- package/dist/parsers/codexParser.js +373 -7
- package/dist/parsers/openCodeParser.js +2 -1
- package/dist/parsers/subagentEndState.d.ts +20 -0
- package/dist/parsers/subagentEndState.js +65 -0
- package/dist/parsers/subagentScanner.d.ts +19 -6
- package/dist/parsers/subagentScanner.js +37 -12
- package/dist/parsers/subagentTraceParser.d.ts +33 -5
- package/dist/parsers/subagentTraceParser.js +103 -37
- package/dist/pricingCatalog.js +3 -2
- package/dist/promptHistory.d.ts +13 -3
- package/dist/promptHistory.js +22 -11
- package/dist/providers/claudeCode.js +59 -28
- package/dist/providers/codex.d.ts +35 -5
- package/dist/providers/codex.js +232 -198
- package/dist/providers/codexCompressedReader.d.ts +58 -0
- package/dist/providers/codexCompressedReader.js +221 -0
- package/dist/providers/codexDatabase.d.ts +25 -0
- package/dist/providers/codexDatabase.js +48 -2
- package/dist/providers/codexRollouts.d.ts +16 -2
- package/dist/providers/codexRollouts.js +20 -2
- package/dist/providers/codexSubagents.d.ts +66 -0
- package/dist/providers/codexSubagents.js +495 -0
- package/dist/providers/compressedRollouts.d.ts +114 -0
- package/dist/providers/compressedRollouts.js +380 -0
- package/dist/providers/factory.d.ts +5 -0
- package/dist/providers/factory.js +4 -1
- package/dist/providers/rolloutWalker.d.ts +15 -2
- package/dist/providers/rolloutWalker.js +33 -4
- package/dist/providers/types.d.ts +36 -1
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/sessionEvent.d.ts +51 -10
- package/dist/schemas/sessionEvent.js +63 -14
- package/dist/sessionMonitor.d.ts +6 -0
- package/dist/sessionMonitor.js +7 -1
- package/dist/sessionPreviews.js +9 -0
- package/dist/sessionPromptHistory.d.ts +5 -0
- package/dist/sessionPromptHistory.js +5 -36
- package/dist/subagentActivity.d.ts +91 -0
- package/dist/subagentActivity.js +132 -0
- package/dist/subagentDisplayState.d.ts +81 -0
- package/dist/subagentDisplayState.js +86 -0
- package/dist/subagentSpawn.d.ts +175 -0
- package/dist/subagentSpawn.js +385 -0
- package/dist/testing/codexSubagentFixtures.d.ts +48 -0
- package/dist/testing/codexSubagentFixtures.js +344 -0
- package/dist/toolCallOrigins.d.ts +5 -0
- package/dist/toolCallOrigins.js +17 -10
- package/dist/toolCategory.js +6 -2
- package/dist/transcript.js +8 -0
- package/dist/turns/assistantTurn.d.ts +4 -0
- package/dist/turns/assistantTurn.js +6 -2
- package/dist/types/codex.d.ts +71 -2
- package/dist/types/sessionEvent.d.ts +38 -3
- package/dist/usage/claudeUsageDedupe.d.ts +31 -0
- package/dist/usage/claudeUsageDedupe.js +68 -0
- package/dist/usage/usageEvents.d.ts +4 -1
- package/dist/usage/usageEvents.js +4 -1
- package/dist/watchers/eventBridge.js +45 -0
- package/dist/watchers/factory.d.ts +6 -0
- package/dist/watchers/factory.js +5 -3
- package/dist/watchers/jsonlTail.d.ts +27 -4
- package/dist/watchers/jsonlTail.js +84 -48
- package/dist/watchers/jsonlWatcher.d.ts +22 -1
- package/dist/watchers/jsonlWatcher.js +60 -3
- package/dist/watchers/providerReaderWatcher.d.ts +7 -0
- package/dist/watchers/providerReaderWatcher.js +18 -4
- package/dist/watchers/types.d.ts +12 -0
- package/dist/zaiQuota.js +2 -1
- package/package.json +1 -1
|
@@ -4,7 +4,9 @@
|
|
|
4
4
|
* Implements SessionProviderBase for monitoring Codex CLI sessions.
|
|
5
5
|
* Codex stores sessions as JSONL rollout files in:
|
|
6
6
|
* ~/.codex/sessions/YYYY/MM/DD/rollout-<timestamp>-<uuid>.jsonl
|
|
7
|
-
* with an optional SQLite index at ~/.codex/state.sqlite.
|
|
7
|
+
* with an optional SQLite index at ~/.codex/state.sqlite. Older rollouts are
|
|
8
|
+
* compressed in place to `….jsonl.zst`; those are read only with the
|
|
9
|
+
* `includeCompressedRollouts` option (see `providers/compressedRollouts`).
|
|
8
10
|
*
|
|
9
11
|
* Ported from sidekick-vscode/src/services/providers/CodexSessionProvider.ts
|
|
10
12
|
*/
|
|
@@ -14,17 +16,20 @@ import { StringDecoder } from 'string_decoder';
|
|
|
14
16
|
import { readSessionContextSnapshot } from '../context/sessionContext';
|
|
15
17
|
import { readSessionFileStats } from '../sessionStats';
|
|
16
18
|
import { CodexRolloutParser } from '../parsers/codexParser';
|
|
17
|
-
import { CodexDatabase
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
19
|
+
import { CodexDatabase } from './codexDatabase';
|
|
20
|
+
import { DEFAULT_ROLLOUT_WALK_MAX_DEPTH, extractRolloutSessionId as extractSessionId, isCompressedRolloutFile, isRolloutFile, walkRolloutFiles, } from './rolloutWalker';
|
|
21
|
+
import { canReadCompressedRollouts, isCompressedRolloutPath, readRolloutHeadAsync, readRolloutText, resolveRolloutPath, } from './compressedRollouts';
|
|
22
|
+
import { CompressedCodexRolloutReader } from './codexCompressedReader';
|
|
23
|
+
import { discoverCodexSubagents } from './codexSubagents';
|
|
24
|
+
import { codexRolloutOrigin } from '../parsers/codexRolloutOrigin';
|
|
20
25
|
import { getCodexSessionsDirs } from '../codexPaths';
|
|
21
26
|
import { cwdMatches, findCodexRolloutFile, getSessionsDirs, listCodexSessionFilesAsync, normalizePath, readFirstLines, readSessionCwd, readSessionCwdAsync, readSessionMeta, } from './codexRollouts';
|
|
22
27
|
import { DirectoryListingCache, splitWatchedPath } from './directoryListingCache';
|
|
28
|
+
import { FileContinuity } from '../fileContinuity';
|
|
23
29
|
import { ProviderDiagnosticTracker, diagnosticFromRuntimeStatus } from './diagnostics';
|
|
24
30
|
import { getModelContextWindowSize } from '../modelContext';
|
|
25
31
|
import { recordObservedContextWindow } from '../observedContextWindows';
|
|
26
|
-
import {
|
|
27
|
-
import { addUsageToSubagent } from '../usage/subagentUsage';
|
|
32
|
+
import { normalizeProviderUsage } from '../usageNormalization';
|
|
28
33
|
// ---------------------------------------------------------------------------
|
|
29
34
|
// Helper functions
|
|
30
35
|
// ---------------------------------------------------------------------------
|
|
@@ -37,8 +42,8 @@ function truncate(text, maxLen) {
|
|
|
37
42
|
return trimmed;
|
|
38
43
|
}
|
|
39
44
|
/** Every rollout under a directory, newest first (one capped walk). */
|
|
40
|
-
function findRolloutFiles(dir) {
|
|
41
|
-
return walkRolloutFiles([dir]);
|
|
45
|
+
function findRolloutFiles(dir, includeCompressed = false) {
|
|
46
|
+
return walkRolloutFiles([dir], { includeCompressed });
|
|
42
47
|
}
|
|
43
48
|
function hasRolloutFile(dir) {
|
|
44
49
|
return walkRolloutFiles([dir], { limit: 1 }).length > 0;
|
|
@@ -47,6 +52,10 @@ function hasRolloutFile(dir) {
|
|
|
47
52
|
function findRolloutFilesInConfiguredHomes(options = {}) {
|
|
48
53
|
return walkRolloutFiles(getSessionsDirs(), options);
|
|
49
54
|
}
|
|
55
|
+
/** Whether a rollout's session_meta says a subagent wrote it. */
|
|
56
|
+
function isSubagentRollout(rolloutPath) {
|
|
57
|
+
return codexRolloutOrigin(readSessionMeta(rolloutPath)).isSubagent;
|
|
58
|
+
}
|
|
50
59
|
export { findCodexRolloutFile };
|
|
51
60
|
/** Pick the best sessions directory for UI/search entrypoints. */
|
|
52
61
|
function getSessionsDir() {
|
|
@@ -124,19 +133,16 @@ function extractFirstUserMessageFromLines(lines) {
|
|
|
124
133
|
return null;
|
|
125
134
|
}
|
|
126
135
|
async function extractFirstUserMessageAsync(rolloutPath) {
|
|
127
|
-
let handle = null;
|
|
128
136
|
try {
|
|
129
|
-
|
|
130
|
-
const
|
|
131
|
-
|
|
132
|
-
|
|
137
|
+
// A compressed rollout's head is decompressed; a plain one is read as is.
|
|
138
|
+
const head = await readRolloutHeadAsync(rolloutPath, 256 * 1024);
|
|
139
|
+
if (head.length === 0)
|
|
140
|
+
return null;
|
|
141
|
+
return extractFirstUserMessageFromLines(head.toString('utf8').split('\n').slice(0, 20));
|
|
133
142
|
}
|
|
134
143
|
catch {
|
|
135
144
|
return null;
|
|
136
145
|
}
|
|
137
|
-
finally {
|
|
138
|
-
await handle?.close().catch(() => undefined);
|
|
139
|
-
}
|
|
140
146
|
}
|
|
141
147
|
/** Extract searchable text from a rollout line. */
|
|
142
148
|
function extractSearchableText(line) {
|
|
@@ -268,50 +274,62 @@ class CodexReader {
|
|
|
268
274
|
/** Cursor after the last complete line, safe to persist across restarts. */
|
|
269
275
|
committedPosition = 0;
|
|
270
276
|
lineBuffer = '';
|
|
271
|
-
|
|
277
|
+
/** Why the last `readNew()` started over from offset 0, if it did. */
|
|
278
|
+
_lastReset = null;
|
|
279
|
+
/**
|
|
280
|
+
* A restart found while settling a seek (the file is now shorter than the
|
|
281
|
+
* position sought); the next `readNew()` reports it.
|
|
282
|
+
*/
|
|
283
|
+
pendingRestart = null;
|
|
284
|
+
/** Whether the rollout is still the one the positions refer to. */
|
|
285
|
+
continuity = new FileContinuity();
|
|
272
286
|
decoder = new StringDecoder('utf8');
|
|
287
|
+
/**
|
|
288
|
+
* Position from `seekTo()` whose prefix still has to be parsed for model,
|
|
289
|
+
* turn, and cumulative-token context; `restoreResumeState()` supplies that
|
|
290
|
+
* context instead and clears it.
|
|
291
|
+
*/
|
|
292
|
+
pendingSeek = null;
|
|
273
293
|
constructor(rolloutPath, onContextWindowLimit, onRateLimits, onTokenUsage) {
|
|
274
294
|
this.rolloutPath = rolloutPath;
|
|
275
295
|
this.onContextWindowLimit = onContextWindowLimit;
|
|
276
296
|
this.onRateLimits = onRateLimits;
|
|
277
297
|
this.onTokenUsage = onTokenUsage;
|
|
278
|
-
this.parser = new CodexRolloutParser();
|
|
298
|
+
this.parser = new CodexRolloutParser({ includeSubagentActivity: true });
|
|
279
299
|
}
|
|
280
300
|
/** Get the parser's session metadata. */
|
|
281
301
|
getSessionMeta() {
|
|
302
|
+
this.settleSeek();
|
|
282
303
|
return this.parser.getSessionMeta();
|
|
283
304
|
}
|
|
284
305
|
readNew(endPosition) {
|
|
285
|
-
this.
|
|
306
|
+
this.settleSeek();
|
|
307
|
+
this._lastReset = null;
|
|
286
308
|
const events = [];
|
|
309
|
+
let fd = null;
|
|
287
310
|
try {
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
const
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
this.
|
|
295
|
-
this.filePosition = 0;
|
|
296
|
-
this.committedPosition = 0;
|
|
297
|
-
this.lineBuffer = '';
|
|
298
|
-
this.parser.reset();
|
|
299
|
-
this.decoder = new StringDecoder('utf8');
|
|
311
|
+
// Open first and stat the descriptor: the size and identity checked
|
|
312
|
+
// are those of the file actually read, even if the path is replaced.
|
|
313
|
+
fd = fs.openSync(this.rolloutPath, 'r');
|
|
314
|
+
const stats = fs.fstatSync(fd);
|
|
315
|
+
if (this.pendingRestart) {
|
|
316
|
+
this._lastReset = this.pendingRestart;
|
|
317
|
+
this.pendingRestart = null;
|
|
300
318
|
}
|
|
319
|
+
const reason = this.continuity.check(fd, stats, this.filePosition, this.committedPosition);
|
|
320
|
+
// Truncated, rewritten, or replaced: what was read no longer describes
|
|
321
|
+
// the rollout, so start over from its beginning.
|
|
322
|
+
if (reason !== null)
|
|
323
|
+
this.restart(reason);
|
|
324
|
+
this.continuity.adopt(stats);
|
|
325
|
+
const currentSize = Math.min(stats.size, endPosition ?? stats.size);
|
|
301
326
|
// No new content
|
|
302
327
|
if (currentSize <= this.filePosition)
|
|
303
|
-
return
|
|
328
|
+
return events;
|
|
304
329
|
// Read new bytes
|
|
305
|
-
const fd = fs.openSync(this.rolloutPath, 'r');
|
|
306
330
|
const bufferSize = currentSize - this.filePosition;
|
|
307
331
|
const buffer = Buffer.alloc(bufferSize);
|
|
308
|
-
|
|
309
|
-
try {
|
|
310
|
-
bytesRead = fs.readSync(fd, buffer, 0, bufferSize, this.filePosition);
|
|
311
|
-
}
|
|
312
|
-
finally {
|
|
313
|
-
fs.closeSync(fd);
|
|
314
|
-
}
|
|
332
|
+
const bytesRead = fs.readSync(fd, buffer, 0, bufferSize, this.filePosition);
|
|
315
333
|
const bytes = buffer.subarray(0, bytesRead);
|
|
316
334
|
const newline = bytes.lastIndexOf(0x0a);
|
|
317
335
|
if (newline >= 0)
|
|
@@ -335,47 +353,71 @@ class CodexReader {
|
|
|
335
353
|
// Skip malformed JSON lines
|
|
336
354
|
}
|
|
337
355
|
}
|
|
338
|
-
|
|
339
|
-
// The model comes along so the value can be persisted per model — the
|
|
340
|
-
// window Codex reports is tier-specific, not the published maximum.
|
|
341
|
-
const mcw = this.parser.getModelContextWindow();
|
|
342
|
-
if (mcw && this.onContextWindowLimit) {
|
|
343
|
-
this.onContextWindowLimit(mcw, this.parser.getCurrentModel() ?? undefined);
|
|
344
|
-
}
|
|
345
|
-
// Propagate rate_limits from token_count events to the provider
|
|
346
|
-
const rateLimits = this.parser.getLastRateLimits();
|
|
347
|
-
if (rateLimits && this.onRateLimits) {
|
|
348
|
-
this.onRateLimits(rateLimits);
|
|
349
|
-
}
|
|
350
|
-
// Propagate last token usage snapshot to the provider
|
|
351
|
-
const lastUsage = this.parser.getLastTokenUsage();
|
|
352
|
-
if (lastUsage && this.onTokenUsage) {
|
|
353
|
-
const normalized = normalizeProviderUsage({
|
|
354
|
-
semantics: 'openai',
|
|
355
|
-
provider: 'openai',
|
|
356
|
-
source: 'codex-reader-snapshot',
|
|
357
|
-
model: this.parser.getCurrentModel() || undefined,
|
|
358
|
-
inputTokens: lastUsage.input_tokens,
|
|
359
|
-
outputTokens: lastUsage.output_tokens,
|
|
360
|
-
cacheReadTokens: lastUsage.cached_input_tokens,
|
|
361
|
-
reasoningTokens: lastUsage.reasoning_output_tokens,
|
|
362
|
-
});
|
|
363
|
-
this.onTokenUsage({
|
|
364
|
-
inputTokens: normalized.uncachedInputTokens,
|
|
365
|
-
outputTokens: normalized.outputTokens,
|
|
366
|
-
cacheWriteTokens: normalized.cacheWriteTokens,
|
|
367
|
-
cacheReadTokens: normalized.cacheReadTokens,
|
|
368
|
-
reasoningTokens: normalized.reasoningTokens,
|
|
369
|
-
model: this.parser.getCurrentModel() || 'unknown',
|
|
370
|
-
timestamp: new Date(),
|
|
371
|
-
});
|
|
372
|
-
}
|
|
356
|
+
this.propagateParserState();
|
|
373
357
|
}
|
|
374
358
|
catch {
|
|
375
|
-
// Skip read errors
|
|
359
|
+
// Skip read errors (a missing rollout reads as no events)
|
|
360
|
+
}
|
|
361
|
+
finally {
|
|
362
|
+
if (fd !== null) {
|
|
363
|
+
try {
|
|
364
|
+
fs.closeSync(fd);
|
|
365
|
+
}
|
|
366
|
+
catch {
|
|
367
|
+
// Ignore close errors.
|
|
368
|
+
}
|
|
369
|
+
}
|
|
376
370
|
}
|
|
377
371
|
return events;
|
|
378
372
|
}
|
|
373
|
+
/** Starts over from offset 0 of a rollout that was truncated, rewritten, or replaced. */
|
|
374
|
+
restart(reason) {
|
|
375
|
+
this._lastReset = { reason, previousPosition: this.committedPosition };
|
|
376
|
+
this.filePosition = 0;
|
|
377
|
+
this.committedPosition = 0;
|
|
378
|
+
this.lineBuffer = '';
|
|
379
|
+
this.parser.reset();
|
|
380
|
+
this.decoder = new StringDecoder('utf8');
|
|
381
|
+
this.pendingSeek = null;
|
|
382
|
+
}
|
|
383
|
+
/** Hands the parser's context window, rate limits, and last usage to the provider. */
|
|
384
|
+
propagateParserState() {
|
|
385
|
+
// Propagate model_context_window from token_count events to the provider.
|
|
386
|
+
// The model comes along so the value can be persisted per model — the
|
|
387
|
+
// window Codex reports is tier-specific, not the published maximum.
|
|
388
|
+
const mcw = this.parser.getModelContextWindow();
|
|
389
|
+
if (mcw && this.onContextWindowLimit) {
|
|
390
|
+
this.onContextWindowLimit(mcw, this.parser.getCurrentModel() ?? undefined);
|
|
391
|
+
}
|
|
392
|
+
// Propagate rate_limits from token_count events to the provider
|
|
393
|
+
const rateLimits = this.parser.getLastRateLimits();
|
|
394
|
+
if (rateLimits && this.onRateLimits) {
|
|
395
|
+
this.onRateLimits(rateLimits);
|
|
396
|
+
}
|
|
397
|
+
// Propagate last token usage snapshot to the provider
|
|
398
|
+
const lastUsage = this.parser.getLastTokenUsage();
|
|
399
|
+
if (lastUsage && this.onTokenUsage) {
|
|
400
|
+
const normalized = normalizeProviderUsage({
|
|
401
|
+
semantics: 'openai',
|
|
402
|
+
provider: 'openai',
|
|
403
|
+
source: 'codex-reader-snapshot',
|
|
404
|
+
model: this.parser.getCurrentModel() || undefined,
|
|
405
|
+
inputTokens: lastUsage.input_tokens,
|
|
406
|
+
outputTokens: lastUsage.output_tokens,
|
|
407
|
+
cacheReadTokens: lastUsage.cached_input_tokens,
|
|
408
|
+
reasoningTokens: lastUsage.reasoning_output_tokens,
|
|
409
|
+
});
|
|
410
|
+
this.onTokenUsage({
|
|
411
|
+
inputTokens: normalized.uncachedInputTokens,
|
|
412
|
+
outputTokens: normalized.outputTokens,
|
|
413
|
+
cacheWriteTokens: normalized.cacheWriteTokens,
|
|
414
|
+
cacheReadTokens: normalized.cacheReadTokens,
|
|
415
|
+
reasoningTokens: normalized.reasoningTokens,
|
|
416
|
+
model: this.parser.getCurrentModel() || 'unknown',
|
|
417
|
+
timestamp: new Date(),
|
|
418
|
+
});
|
|
419
|
+
}
|
|
420
|
+
}
|
|
379
421
|
readAll() {
|
|
380
422
|
this.reset();
|
|
381
423
|
return this.readNew();
|
|
@@ -386,12 +428,17 @@ class CodexReader {
|
|
|
386
428
|
this.lineBuffer = '';
|
|
387
429
|
this.parser.reset();
|
|
388
430
|
this.decoder = new StringDecoder('utf8');
|
|
389
|
-
this.
|
|
431
|
+
this._lastReset = null;
|
|
432
|
+
this.pendingRestart = null;
|
|
433
|
+
this.pendingSeek = null;
|
|
434
|
+
// The next read adopts whatever rollout has the path.
|
|
435
|
+
this.continuity.forget();
|
|
390
436
|
}
|
|
391
437
|
exists() {
|
|
392
438
|
return fs.existsSync(this.rolloutPath);
|
|
393
439
|
}
|
|
394
440
|
flush() {
|
|
441
|
+
this.settleSeek();
|
|
395
442
|
this.lineBuffer += this.decoder.end();
|
|
396
443
|
// Process any remaining data in the line buffer
|
|
397
444
|
if (this.lineBuffer.trim()) {
|
|
@@ -409,19 +456,55 @@ class CodexReader {
|
|
|
409
456
|
return this.committedPosition;
|
|
410
457
|
}
|
|
411
458
|
seekTo(position) {
|
|
412
|
-
// Rebuild model, turn, and cumulative-token context before resuming.
|
|
413
|
-
// The prefix is parsed only; its events are already in the consumer snapshot.
|
|
414
459
|
this.reset();
|
|
460
|
+
this.filePosition = position;
|
|
461
|
+
this.committedPosition = position;
|
|
462
|
+
// The prefix's context is rebuilt on the next read, unless
|
|
463
|
+
// restoreResumeState() supplies it first.
|
|
415
464
|
if (position > 0)
|
|
416
|
-
this.
|
|
465
|
+
this.pendingSeek = position;
|
|
417
466
|
}
|
|
418
467
|
wasTruncated() {
|
|
419
|
-
return this.
|
|
468
|
+
return this._lastReset !== null;
|
|
469
|
+
}
|
|
470
|
+
lastReset() {
|
|
471
|
+
return this._lastReset ? { ...this._lastReset } : null;
|
|
472
|
+
}
|
|
473
|
+
/** The parser's continuation state (see `CodexRolloutParser.exportResumeState`). */
|
|
474
|
+
getResumeState() {
|
|
475
|
+
this.settleSeek();
|
|
476
|
+
return this.parser.exportResumeState();
|
|
477
|
+
}
|
|
478
|
+
/**
|
|
479
|
+
* Restores parser state from `getResumeState()`; after `seekTo()` this
|
|
480
|
+
* replaces re-parsing the rollout up to that position.
|
|
481
|
+
*/
|
|
482
|
+
restoreResumeState(state) {
|
|
483
|
+
if (!this.parser.importResumeState(state))
|
|
484
|
+
return false;
|
|
485
|
+
this.pendingSeek = null;
|
|
486
|
+
this.propagateParserState();
|
|
487
|
+
return true;
|
|
488
|
+
}
|
|
489
|
+
/**
|
|
490
|
+
* Rebuilds model, turn, and cumulative-token context for a pending
|
|
491
|
+
* `seekTo()` position. The prefix is parsed only; its events are already
|
|
492
|
+
* in the consumer snapshot.
|
|
493
|
+
*/
|
|
494
|
+
settleSeek() {
|
|
495
|
+
const position = this.pendingSeek;
|
|
496
|
+
if (position === null)
|
|
497
|
+
return;
|
|
498
|
+
this.reset();
|
|
499
|
+
this.readNew(position);
|
|
500
|
+
if (this.filePosition < position) {
|
|
501
|
+
// The rollout no longer reaches the position sought: it was truncated
|
|
502
|
+
// or replaced since. The next read starts over and reports it.
|
|
503
|
+
this.reset();
|
|
504
|
+
this.pendingRestart = { reason: 'truncated', previousPosition: position };
|
|
505
|
+
}
|
|
420
506
|
}
|
|
421
507
|
}
|
|
422
|
-
// ---------------------------------------------------------------------------
|
|
423
|
-
// CodexProvider — full SessionProviderBase implementation
|
|
424
|
-
// ---------------------------------------------------------------------------
|
|
425
508
|
/**
|
|
426
509
|
* Session provider for Codex CLI.
|
|
427
510
|
*
|
|
@@ -440,8 +523,26 @@ export class CodexProvider {
|
|
|
440
523
|
lastTokenUsageData = null;
|
|
441
524
|
/** Dated rollout-directory listings reused across async enumerations. */
|
|
442
525
|
directoryCache = new DirectoryListingCache();
|
|
526
|
+
/** See {@link CodexProviderOptions.includeCompressedRollouts}. */
|
|
527
|
+
compressedRequested;
|
|
528
|
+
onDiagnostic;
|
|
443
529
|
constructor(options = {}) {
|
|
444
530
|
this.diagnostics = new ProviderDiagnosticTracker(this.id, options);
|
|
531
|
+
this.compressedRequested = options.includeCompressedRollouts === true;
|
|
532
|
+
this.onDiagnostic = options.onDiagnostic;
|
|
533
|
+
}
|
|
534
|
+
/**
|
|
535
|
+
* Whether compressed rollouts are discovered and read: requested, and the
|
|
536
|
+
* runtime can decompress them (otherwise reported once, on first use).
|
|
537
|
+
*/
|
|
538
|
+
get includeCompressed() {
|
|
539
|
+
return (this.compressedRequested && canReadCompressedRollouts({ onDiagnostic: this.onDiagnostic }));
|
|
540
|
+
}
|
|
541
|
+
/** A stored rollout path as it exists now (its `.zst` when compressed and enabled). */
|
|
542
|
+
resolveRollout(rolloutPath) {
|
|
543
|
+
return rolloutPath
|
|
544
|
+
? resolveRolloutPath(rolloutPath, { includeCompressed: this.includeCompressed })
|
|
545
|
+
: null;
|
|
445
546
|
}
|
|
446
547
|
// --- Database ---
|
|
447
548
|
/** Lazy-initialize database connection. */
|
|
@@ -539,10 +640,11 @@ export class CodexProvider {
|
|
|
539
640
|
return thread.rollout_path;
|
|
540
641
|
}
|
|
541
642
|
}
|
|
542
|
-
// File scan fallback (newest first; cwd reads are cached per path)
|
|
643
|
+
// File scan fallback (newest first; cwd reads are cached per path).
|
|
644
|
+
// Subagent rollouts share their parent's cwd but are not the session.
|
|
543
645
|
for (const file of findRolloutFilesInConfiguredHomes()) {
|
|
544
646
|
const cwd = readSessionCwd(file.path);
|
|
545
|
-
if (cwd && cwdMatches(cwd, workspacePath)) {
|
|
647
|
+
if (cwd && cwdMatches(cwd, workspacePath) && !isSubagentRollout(file.path)) {
|
|
546
648
|
this.recordDatabaseFallback('findActiveSession', true);
|
|
547
649
|
return file.path;
|
|
548
650
|
}
|
|
@@ -556,15 +658,17 @@ export class CodexProvider {
|
|
|
556
658
|
if (db) {
|
|
557
659
|
const threads = db.getThreadsByCwd(workspacePath);
|
|
558
660
|
const dbPaths = threads
|
|
559
|
-
.
|
|
560
|
-
.
|
|
661
|
+
.map((t) => this.resolveRollout(t.rollout_path))
|
|
662
|
+
.filter((rolloutPath) => rolloutPath !== null);
|
|
561
663
|
if (dbPaths.length > 0) {
|
|
562
664
|
this.recordDatabaseFallback('findAllSessions', true);
|
|
563
665
|
return dbPaths;
|
|
564
666
|
}
|
|
565
667
|
}
|
|
566
668
|
// File scan (newest first; cwd reads are cached per path)
|
|
567
|
-
const sessions = findRolloutFilesInConfiguredHomes(
|
|
669
|
+
const sessions = findRolloutFilesInConfiguredHomes({
|
|
670
|
+
includeCompressed: this.includeCompressed,
|
|
671
|
+
})
|
|
568
672
|
.filter((f) => {
|
|
569
673
|
const cwd = readSessionCwd(f.path);
|
|
570
674
|
return cwd !== null && cwdMatches(cwd, workspacePath);
|
|
@@ -575,7 +679,7 @@ export class CodexProvider {
|
|
|
575
679
|
}
|
|
576
680
|
findSessionById(_workspacePath, sessionId) {
|
|
577
681
|
try {
|
|
578
|
-
const found = findCodexRolloutFile(sessionId);
|
|
682
|
+
const found = findCodexRolloutFile(sessionId, { includeCompressed: this.includeCompressed });
|
|
579
683
|
this.recordDatabaseFallback('findSessionById', true);
|
|
580
684
|
return found;
|
|
581
685
|
}
|
|
@@ -589,15 +693,15 @@ export class CodexProvider {
|
|
|
589
693
|
return this.findAllSessions(workspacePath);
|
|
590
694
|
}
|
|
591
695
|
findSessionsInDirectory(dir) {
|
|
592
|
-
return findRolloutFiles(dir).map((f) => f.path);
|
|
696
|
+
return findRolloutFiles(dir, this.includeCompressed).map((f) => f.path);
|
|
593
697
|
}
|
|
594
698
|
listAllSessionFiles() {
|
|
595
|
-
const files = findRolloutFilesInConfiguredHomes();
|
|
699
|
+
const files = findRolloutFilesInConfiguredHomes({ includeCompressed: this.includeCompressed });
|
|
596
700
|
this.recordDatabaseFallback('listAllSessionFiles', true);
|
|
597
701
|
return files;
|
|
598
702
|
}
|
|
599
703
|
async listSessionFilesAsync(workspacePath, options = {}) {
|
|
600
|
-
const results = await listCodexSessionFilesAsync(workspacePath, options, this.directoryCache);
|
|
704
|
+
const results = await listCodexSessionFilesAsync(workspacePath, { ...options, includeCompressed: this.includeCompressed }, this.directoryCache);
|
|
601
705
|
// Filesystem enumeration bypasses the database, so record that here as
|
|
602
706
|
// the sync listing paths do; otherwise `getLastOperationStatus()` keeps
|
|
603
707
|
// describing whichever operation ran before this one.
|
|
@@ -617,7 +721,7 @@ export class CodexProvider {
|
|
|
617
721
|
const name = segments[segments.length - 1];
|
|
618
722
|
if (segments.length - 1 > DEFAULT_ROLLOUT_WALK_MAX_DEPTH)
|
|
619
723
|
return { status: 'ignored' };
|
|
620
|
-
if (!
|
|
724
|
+
if (!this.isSessionFile(name)) {
|
|
621
725
|
this.directoryCache.invalidate(fullPath);
|
|
622
726
|
try {
|
|
623
727
|
const stat = await fs.promises.stat(fullPath);
|
|
@@ -667,7 +771,9 @@ export class CodexProvider {
|
|
|
667
771
|
// after its last update can be missing from it, so older files are not
|
|
668
772
|
// opened; the cwd read itself is cached per path either way.
|
|
669
773
|
const indexedUpTo = db ? db.getDbMtime() : 0;
|
|
670
|
-
for (const file of findRolloutFilesInConfiguredHomes(
|
|
774
|
+
for (const file of findRolloutFilesInConfiguredHomes({
|
|
775
|
+
includeCompressed: this.includeCompressed,
|
|
776
|
+
})) {
|
|
671
777
|
if (db && indexedUpTo > 0 && file.mtime.getTime() <= indexedUpTo)
|
|
672
778
|
continue;
|
|
673
779
|
const cwd = readSessionCwd(file.path);
|
|
@@ -709,7 +815,7 @@ export class CodexProvider {
|
|
|
709
815
|
}
|
|
710
816
|
// --- File identification ---
|
|
711
817
|
isSessionFile(filename) {
|
|
712
|
-
return isRolloutFile(filename);
|
|
818
|
+
return isRolloutFile(filename) || (this.includeCompressed && isCompressedRolloutFile(filename));
|
|
713
819
|
}
|
|
714
820
|
getSessionId(sessionPath) {
|
|
715
821
|
return extractSessionId(path.basename(sessionPath));
|
|
@@ -770,6 +876,17 @@ export class CodexProvider {
|
|
|
770
876
|
}
|
|
771
877
|
// --- Data reading ---
|
|
772
878
|
createReader(sessionPath) {
|
|
879
|
+
if (isCompressedRolloutPath(sessionPath)) {
|
|
880
|
+
// Compressed rollouts are complete: read whole, never tailed.
|
|
881
|
+
return new CompressedCodexRolloutReader(sessionPath, {
|
|
882
|
+
onContextWindowLimit: (limit) => {
|
|
883
|
+
// Historical content: keep the live limit, do not persist it again.
|
|
884
|
+
if (this.dynamicContextWindowLimit === null)
|
|
885
|
+
this.dynamicContextWindowLimit = limit;
|
|
886
|
+
},
|
|
887
|
+
onDiagnostic: this.onDiagnostic,
|
|
888
|
+
});
|
|
889
|
+
}
|
|
773
890
|
return new CodexReader(sessionPath, (limit, modelId) => {
|
|
774
891
|
this.dynamicContextWindowLimit = limit;
|
|
775
892
|
if (modelId) {
|
|
@@ -786,119 +903,36 @@ export class CodexProvider {
|
|
|
786
903
|
});
|
|
787
904
|
}
|
|
788
905
|
// --- Subagent support ---
|
|
789
|
-
scanSubagents(_sessionDir, sessionId) {
|
|
790
|
-
// Codex tracks forked sessions via forked_from_id in session_meta.
|
|
791
|
-
// Use DB when available, fall back to filesystem scanning.
|
|
792
|
-
const subagents = [];
|
|
793
|
-
// DB approach: spawned subagents (parent recorded in `threads.source`),
|
|
794
|
-
// plus forked children on older schemas that carry `forked_from_id`.
|
|
795
|
-
const db = this.ensureDb();
|
|
796
|
-
if (db) {
|
|
797
|
-
const seen = new Set();
|
|
798
|
-
const children = [
|
|
799
|
-
...db.getSpawnedChildThreads(sessionId),
|
|
800
|
-
...db.getThreadsByForkedFromId(sessionId),
|
|
801
|
-
].filter((thread) => !seen.has(thread.id) && seen.add(thread.id));
|
|
802
|
-
for (const thread of children) {
|
|
803
|
-
const built = cachedSubagentStats(thread.rollout_path, () => this.buildSubagentStats(thread.id, thread.rollout_path, thread.title || thread.agent_nickname));
|
|
804
|
-
if (built) {
|
|
805
|
-
subagents.push(thread.agent_role && !built.agentType
|
|
806
|
-
? { ...built, agentType: thread.agent_role }
|
|
807
|
-
: built);
|
|
808
|
-
}
|
|
809
|
-
}
|
|
810
|
-
if (children.length > 0)
|
|
811
|
-
return subagents;
|
|
812
|
-
}
|
|
813
|
-
// Filesystem fallback: scan recent rollout files for a matching
|
|
814
|
-
// forked_from_id. The walk stops after the 50 newest-dated files instead
|
|
815
|
-
// of enumerating the whole history and slicing afterwards.
|
|
816
|
-
for (const file of findRolloutFilesInConfiguredHomes({ limit: 50 })) {
|
|
817
|
-
const meta = readSessionMeta(file.path);
|
|
818
|
-
if (meta &&
|
|
819
|
-
(meta.forked_from_id === sessionId || spawnParentFromSource(meta.source) === sessionId)) {
|
|
820
|
-
const childId = extractSessionId(path.basename(file.path));
|
|
821
|
-
// `source` is a plain string for top-level sessions but an object
|
|
822
|
-
// (`{ subagent: { thread_spawn: … } }`) for spawned children.
|
|
823
|
-
const description = typeof meta.source === 'string' ? meta.source : undefined;
|
|
824
|
-
const stats = cachedSubagentStats(file.path, () => this.buildSubagentStats(childId, file.path, description));
|
|
825
|
-
if (stats)
|
|
826
|
-
subagents.push(stats);
|
|
827
|
-
}
|
|
828
|
-
}
|
|
829
|
-
return subagents;
|
|
830
|
-
}
|
|
831
906
|
/**
|
|
832
|
-
*
|
|
907
|
+
* Every subagent below the session, any depth: spawned threads (the
|
|
908
|
+
* database's `threads.source`, else the newest rollouts' session_meta)
|
|
909
|
+
* and, on older schemas, forks (`forked_from_id`). Each carries its
|
|
910
|
+
* rollout (`filePath`), the parent's spawn call (`spawnToolUseId`),
|
|
911
|
+
* `agentNickname`, `agentPath`, `parentAgentId`, and `depth`; tool calls
|
|
912
|
+
* and tokens come from the child's own records, not the parent history a
|
|
913
|
+
* forked child embeds.
|
|
833
914
|
*/
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
const content = fs.readFileSync(rolloutPath, 'utf8');
|
|
847
|
-
const lines = content.split('\n');
|
|
848
|
-
for (const line of lines) {
|
|
849
|
-
const trimmed = line.trim();
|
|
850
|
-
if (!trimmed)
|
|
851
|
-
continue;
|
|
852
|
-
try {
|
|
853
|
-
const parsed = JSON.parse(trimmed);
|
|
854
|
-
// Track timestamps
|
|
855
|
-
if (parsed.timestamp) {
|
|
856
|
-
const ts = new Date(parsed.timestamp);
|
|
857
|
-
if (!stats.startTime || ts < stats.startTime)
|
|
858
|
-
stats.startTime = ts;
|
|
859
|
-
if (!stats.endTime || ts > stats.endTime)
|
|
860
|
-
stats.endTime = ts;
|
|
861
|
-
}
|
|
862
|
-
const events = parser.convertLine(parsed);
|
|
863
|
-
for (const event of events) {
|
|
864
|
-
// Collect tool calls
|
|
865
|
-
if (event.type === 'assistant' && Array.isArray(event.message.content)) {
|
|
866
|
-
for (const block of event.message.content) {
|
|
867
|
-
if (block.type === 'tool_use' && typeof block.name === 'string') {
|
|
868
|
-
stats.toolCalls.push({
|
|
869
|
-
name: block.name,
|
|
870
|
-
input: block.input || {},
|
|
871
|
-
timestamp: new Date(event.timestamp),
|
|
872
|
-
});
|
|
873
|
-
}
|
|
874
|
-
}
|
|
875
|
-
}
|
|
876
|
-
// Accumulate tokens from usage events (duplicate token_count events
|
|
877
|
-
// are already dropped by the parser).
|
|
878
|
-
const usage = extractNormalizedUsage(event);
|
|
879
|
-
if (usage)
|
|
880
|
-
addUsageToSubagent(stats, usage);
|
|
881
|
-
}
|
|
882
|
-
}
|
|
883
|
-
catch {
|
|
884
|
-
/* skip malformed lines */
|
|
885
|
-
}
|
|
886
|
-
}
|
|
887
|
-
if (stats.startTime && stats.endTime) {
|
|
888
|
-
stats.durationMs = stats.endTime.getTime() - stats.startTime.getTime();
|
|
889
|
-
}
|
|
890
|
-
}
|
|
891
|
-
catch {
|
|
892
|
-
/* skip unreadable files */
|
|
893
|
-
}
|
|
894
|
-
return stats;
|
|
915
|
+
scanSubagents(_sessionDir, sessionId) {
|
|
916
|
+
const db = this.ensureDb();
|
|
917
|
+
return discoverCodexSubagents(sessionId, {
|
|
918
|
+
db,
|
|
919
|
+
includeCompressed: this.includeCompressed,
|
|
920
|
+
onDiagnostic: this.onDiagnostic,
|
|
921
|
+
sessionRolloutPath: () => this.resolveRollout(db?.getThread(sessionId)?.rollout_path) ??
|
|
922
|
+
findCodexRolloutFile(sessionId, { includeCompressed: this.includeCompressed }),
|
|
923
|
+
// The walk stops after the 50 newest-dated files instead of
|
|
924
|
+
// enumerating the whole history and slicing afterwards.
|
|
925
|
+
fallbackFiles: () => findRolloutFilesInConfiguredHomes({ limit: 50, includeCompressed: this.includeCompressed }),
|
|
926
|
+
});
|
|
895
927
|
}
|
|
896
928
|
// --- Cross-session search ---
|
|
897
929
|
searchInSession(sessionPath, query, maxResults) {
|
|
898
930
|
const results = [];
|
|
899
931
|
const queryLower = query.toLowerCase();
|
|
900
932
|
try {
|
|
901
|
-
const content =
|
|
933
|
+
const content = readRolloutText(sessionPath, { onDiagnostic: this.onDiagnostic });
|
|
934
|
+
if (content === null)
|
|
935
|
+
return results;
|
|
902
936
|
const lines = content.split('\n');
|
|
903
937
|
for (const line of lines) {
|
|
904
938
|
if (results.length >= maxResults)
|