taskchef 9.0.3 → 9.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.codex-plugin/plugin.json +1 -1
- package/docs/mcp-app.md +6 -0
- package/docs/taskchef-next-storage.md +10 -0
- package/package.json +1 -1
- package/src/codex-projects.js +4 -2
- package/src/codex-rollout.js +36 -0
- package/src/codex-session-scanner.js +38 -19
- package/src/codex-session-usage.js +50 -48
- package/src/mcp-app/dist/index.html +77 -72
- package/src/mcp-app/react/main.test.tsx +35 -1
- package/src/mcp-app/react/main.tsx +6 -3
- package/src/mcp-app/react/styles.css +6 -2
- package/src/ui/types.ts +1 -1
package/docs/mcp-app.md
CHANGED
|
@@ -298,3 +298,9 @@ its cards, controls, borders, and elevated surfaces from those seeds. Links and
|
|
|
298
298
|
small muted text are adjusted when needed to meet a 4.5:1 contrast ratio on the
|
|
299
299
|
UI surfaces. Native font families use locally available fonts with system fallbacks. Font assets are not copied from the desktop application. Palette definitions ship with TaskChef; opening the board does not
|
|
300
300
|
read the installed desktop application or fetch theme files from the network.
|
|
301
|
+
|
|
302
|
+
The compact **Pinned** toggle button shows chats pinned in Codex and combines with project,
|
|
303
|
+
update window, and search filters. It starts off when the dashboard opens. Pin
|
|
304
|
+
changes appear on the next data refresh. The current desktop pin list takes
|
|
305
|
+
precedence over the database `is_pinned` field. TaskChef ignores old migration
|
|
306
|
+
snapshots and never writes pin state. Without either source, chats are unpinned.
|
|
@@ -88,6 +88,16 @@ not read again. Appended files are read from the last complete line; replaced
|
|
|
88
88
|
or truncated files are read again. Changed compressed files are decoded again;
|
|
89
89
|
unchanged compressed files use the same cache. A runtime without zstd support
|
|
90
90
|
marks compressed histories partial. Concurrent views share this reader.
|
|
91
|
+
Plain and compressed siblings share one cache identity. Reads prefer the plain
|
|
92
|
+
file when both exist and retry briefly during format transitions. A stale
|
|
93
|
+
selected path can resolve its sibling, so compression or restoration alone does
|
|
94
|
+
not mark accounting partial. Existing compressed cache identities are normalized
|
|
95
|
+
when loaded.
|
|
96
|
+
|
|
97
|
+
Optional Details message counts use the same path resolution. Compressed logs
|
|
98
|
+
are decoded as a stream, retaining bounded head and tail samples. Their
|
|
99
|
+
`fileBytes` and `sampledBytes` describe decoded JSONL bytes, as for plain files.
|
|
100
|
+
Compression does not change database-derived board status or replies.
|
|
91
101
|
|
|
92
102
|
Computed accounting results are saved under TaskChef's state directory in
|
|
93
103
|
`cache/usage/`, with one hashed file per chat and duration-view variant. Files
|
package/package.json
CHANGED
package/src/codex-projects.js
CHANGED
|
@@ -8,7 +8,7 @@ const contains = (root, cwd) => {
|
|
|
8
8
|
return child === "" || (!child.startsWith(`..${sep}`) && child !== ".." && !isAbsolute(child));
|
|
9
9
|
};
|
|
10
10
|
|
|
11
|
-
// Only project metadata and unread chat IDs leave this reader. Never return the desktop's other saved settings.
|
|
11
|
+
// Only project metadata, pin state, and unread chat IDs leave this reader. Never return the desktop's other saved settings.
|
|
12
12
|
export class CodexProjects {
|
|
13
13
|
constructor(codexHome) { this.codexHome = codexHome; this.saved = null; }
|
|
14
14
|
async desktopProjects() {
|
|
@@ -20,7 +20,9 @@ export class CodexProjects {
|
|
|
20
20
|
} catch (error) { if (error.code === "ENOENT") return {}; throw error; }
|
|
21
21
|
if (this.saved?.stamp === stamp) return this.saved.data;
|
|
22
22
|
const state = JSON.parse(await readFile(file, "utf8"));
|
|
23
|
+
const pinned = state["electron-persisted-atom-state"]?.["app-server-pinned-thread-order-v1"];
|
|
23
24
|
const data = {
|
|
25
|
+
pinned: Array.isArray(pinned) ? new Set(pinned.filter(id => typeof id === "string")) : null,
|
|
24
26
|
unread: localUnreadChats(state["electron-thread-read-state-v1"]),
|
|
25
27
|
assignments: state["thread-project-assignments"] ?? {},
|
|
26
28
|
hints: state["thread-workspace-root-hints"] ?? {}, projectless: new Set(state["projectless-thread-ids"] ?? []),
|
|
@@ -71,7 +73,7 @@ export class CodexProjects {
|
|
|
71
73
|
if (!explicit && !noProject && !hinted && !origins.has(cwd)) origins.set(cwd, this.worktreeRoot(cwd));
|
|
72
74
|
const origin = origins.has(cwd) ? await origins.get(cwd) : null;
|
|
73
75
|
const project = noProject ? null : explicit ?? hinted ?? matchRoot(origin || cwd);
|
|
74
|
-
return { ...task, ...(task.observed ? { observed: { ...task.observed, unread: saved.unread?.has(task.id) ?? false } } : {}), project: { ...task.project, id: project?.id ?? NO_PROJECT, name: project?.name ?? "No project" } };
|
|
76
|
+
return { ...task, ...(task.observed ? { observed: { ...task.observed, pinned: saved.pinned ? saved.pinned.has(task.id) : task.observed.pinned ?? false, unread: saved.unread?.has(task.id) ?? false } } : {}), project: { ...task.project, id: project?.id ?? NO_PROJECT, name: project?.name ?? "No project" } };
|
|
75
77
|
}));
|
|
76
78
|
return { tasks: grouped, projects: projects.map(({ id, name, roots }) => ({ id, name, path: roots[0] ?? "", githubRepos: [] })) };
|
|
77
79
|
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { open, stat } from "node:fs/promises";
|
|
2
|
+
import * as zlib from "node:zlib";
|
|
3
|
+
|
|
4
|
+
export function plainRolloutPath(path) { return path.replace(/\.jsonl\.zst$/, ".jsonl"); }
|
|
5
|
+
|
|
6
|
+
// Codex can replace either representation while readers are resolving a path.
|
|
7
|
+
async function resolve(path, inspect) {
|
|
8
|
+
const plain = plainRolloutPath(path);
|
|
9
|
+
for (let attempt = 0; ; attempt++) {
|
|
10
|
+
for (const candidate of [plain, `${plain}.zst`]) {
|
|
11
|
+
try { return { path: candidate, ...await inspect(candidate) }; }
|
|
12
|
+
catch (error) { if (error.code !== "ENOENT") throw error; }
|
|
13
|
+
}
|
|
14
|
+
if (attempt === 3) { const error = new Error("Rollout is missing."); error.code = "ENOENT"; throw error; }
|
|
15
|
+
await new Promise(resolve => setTimeout(resolve, 50));
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function rolloutMetadata(path) { return resolve(path, async path => ({ info: await stat(path) })); }
|
|
20
|
+
export function openRollout(path) {
|
|
21
|
+
return resolve(path, async path => {
|
|
22
|
+
const handle = await open(path, "r");
|
|
23
|
+
try { return { handle, info: await handle.stat() }; }
|
|
24
|
+
catch (error) { await handle.close(); throw error; }
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export async function* rolloutChunks({ path, handle }, options = {}) {
|
|
29
|
+
const compressed = path.endsWith(".jsonl.zst");
|
|
30
|
+
if (compressed && typeof zlib.createZstdDecompress !== "function") throw new Error("This runtime cannot decode compressed rollouts.");
|
|
31
|
+
const input = handle.createReadStream({ ...options, autoClose: false });
|
|
32
|
+
const stream = compressed ? zlib.createZstdDecompress() : input;
|
|
33
|
+
if (compressed) { input.on("error", error => stream.destroy(error)); input.pipe(stream); }
|
|
34
|
+
try { for await (const chunk of stream) yield chunk; }
|
|
35
|
+
finally { input.destroy(); if (compressed) stream.destroy(); }
|
|
36
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { openRollout, rolloutChunks } from "./codex-rollout.js";
|
|
1
2
|
import { createHash } from "node:crypto";
|
|
2
3
|
import { normalizeUri } from "micromark-util-sanitize-uri";
|
|
3
4
|
import { displayUserPrompt, responseAnnotations } from "./user-prompt.js";
|
|
@@ -7,7 +8,7 @@ import { CodexPricing } from "./codex-pricing.js";
|
|
|
7
8
|
import { replyImage, replyImages, localReplyImage } from "./reply-image.js";
|
|
8
9
|
import { homedir } from "node:os";
|
|
9
10
|
import { basename, isAbsolute, join } from "node:path";
|
|
10
|
-
import {
|
|
11
|
+
import { stat, readFile, readdir, mkdir } from "node:fs/promises";
|
|
11
12
|
import { acquireFileLock } from "./file-lock.js";
|
|
12
13
|
import { writeDurableAtomic } from "./state-store.js";
|
|
13
14
|
import { dirname } from "node:path";
|
|
@@ -232,12 +233,13 @@ function readDatabaseRecords(state, history) {
|
|
|
232
233
|
const eligible = `coalesce(thread_source, '') NOT IN ('subagent', 'guardian_review')
|
|
233
234
|
AND CASE WHEN json_valid(source) THEN json_type(source, '$.subagent') IS NULL ELSE 1 END
|
|
234
235
|
AND NOT EXISTS (SELECT 1 FROM thread_spawn_edges WHERE child_thread_id = threads.id)`;
|
|
236
|
+
const hasPinned = state.prepare("PRAGMA table_info(threads)").all().some(column => column.name === "is_pinned");
|
|
235
237
|
const hasTurnError = history.prepare("PRAGMA table_info(thread_turns)").all().some(column => column.name === "error_json");
|
|
236
238
|
const latestTurn = history.prepare(`SELECT turn_id, status, started_at, duration_ms, first_user_item_id, final_agent_item_id, ${hasTurnError ? "error_json" : "NULL AS error_json"}
|
|
237
239
|
FROM thread_turns WHERE thread_id = ? ORDER BY rollout_ordinal DESC LIMIT 1`);
|
|
238
240
|
const turns = new Map();
|
|
239
241
|
const needsAnswers = new Map();
|
|
240
|
-
const rows = state.prepare(`SELECT id, name, title, cwd, archived, source, created_at_ms, updated_at_ms, recency_at_ms, rollout_path, history_mode, project_id,
|
|
242
|
+
const rows = state.prepare(`SELECT id, name, title, cwd, archived, ${hasPinned ? "is_pinned" : "0 AS is_pinned"}, source, created_at_ms, updated_at_ms, recency_at_ms, rollout_path, history_mode, project_id,
|
|
241
243
|
(SELECT count(*) FROM thread_spawn_edges WHERE parent_thread_id = threads.id) AS child_count
|
|
242
244
|
FROM threads WHERE ${eligible} ORDER BY recency_at_ms DESC`).all().filter((row) => {
|
|
243
245
|
const key = historyId(row);
|
|
@@ -360,7 +362,7 @@ function buildDatabaseSnapshot(records, now, doneMarks, schedules, scheduleError
|
|
|
360
362
|
createdAt: iso(row.created_at_ms || updatedMs), updatedAt: iso(updatedMs),
|
|
361
363
|
updatedBy: "Local Codex database", project: { id: row.project_id, name: basename(cwd) || cwd || "Unknown project", path: cwd, githubRepos: [] },
|
|
362
364
|
threadId: row.id, turnRef: turn.turn_id, turnId: turn.turn_id, lastResult: null, latestTurn: null,
|
|
363
|
-
observed: { source: row.source, archive: Boolean(row.archived), lastTurnEvent: turn.status, lastTurnEventAt: turn.started_at ? iso(turn.started_at * 1000) : null, latestTurnDurationMs: turn.status !== "inProgress" && Number.isFinite(turn.duration_ms) && turn.duration_ms >= 0 ? turn.duration_ms : null, recentFileActivity: !row.archived && now - updatedMs < ACTIVE_WINDOW_MS, directChildCount: row.child_count },
|
|
365
|
+
observed: { pinned: Boolean(row.is_pinned), source: row.source, archive: Boolean(row.archived), lastTurnEvent: turn.status, lastTurnEventAt: turn.started_at ? iso(turn.started_at * 1000) : null, latestTurnDurationMs: turn.status !== "inProgress" && Number.isFinite(turn.duration_ms) && turn.duration_ms >= 0 ? turn.duration_ms : null, recentFileActivity: !row.archived && now - updatedMs < ACTIVE_WINDOW_MS, directChildCount: row.child_count },
|
|
364
366
|
};
|
|
365
367
|
});
|
|
366
368
|
return { tasks, rolloutPaths: new Map(rows.map((row) => [row.id, row.rollout_path])), scan: {
|
|
@@ -392,18 +394,37 @@ async function readSlice(handle, start, bytes) {
|
|
|
392
394
|
}
|
|
393
395
|
|
|
394
396
|
async function readSession(file, now) {
|
|
395
|
-
const
|
|
397
|
+
const source = await openRollout(file.path);
|
|
398
|
+
const { handle } = source;
|
|
396
399
|
try {
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
400
|
+
let size, headEnd, head, tail, contiguous;
|
|
401
|
+
if (source.path.endsWith(".jsonl.zst")) {
|
|
402
|
+
let prefix = Buffer.alloc(0), suffix = Buffer.alloc(0);
|
|
403
|
+
size = 0;
|
|
404
|
+
for await (const chunk of rolloutChunks(source, { highWaterMark: TAIL_BYTES })) {
|
|
405
|
+
size += chunk.length;
|
|
406
|
+
if (prefix.length < META_LINE_BYTES) prefix = Buffer.concat([prefix, chunk.subarray(0, META_LINE_BYTES - prefix.length)]);
|
|
407
|
+
suffix = Buffer.concat([suffix, chunk]);
|
|
408
|
+
if (suffix.length > TAIL_BYTES) suffix = Buffer.from(suffix.subarray(suffix.length - TAIL_BYTES));
|
|
409
|
+
}
|
|
410
|
+
headEnd = Math.min(size, HEAD_BYTES);
|
|
411
|
+
if (!prefix.subarray(0, headEnd).includes(10) && headEnd < size) headEnd = Math.min(size, META_LINE_BYTES);
|
|
412
|
+
head = prefix.subarray(0, headEnd).toString("utf8");
|
|
413
|
+
const tailStart = Math.max(0, size - TAIL_BYTES);
|
|
414
|
+
contiguous = tailStart <= headEnd;
|
|
415
|
+
tail = suffix.subarray(contiguous ? headEnd - tailStart : 0).toString("utf8");
|
|
416
|
+
} else {
|
|
417
|
+
size = source.info.size;
|
|
418
|
+
headEnd = Math.min(size, HEAD_BYTES);
|
|
402
419
|
head = await readSlice(handle, 0, headEnd);
|
|
420
|
+
if (!head.includes("\n") && headEnd < size) {
|
|
421
|
+
headEnd = Math.min(size, META_LINE_BYTES);
|
|
422
|
+
head = await readSlice(handle, 0, headEnd);
|
|
423
|
+
}
|
|
424
|
+
const tailStart = Math.max(0, size - TAIL_BYTES);
|
|
425
|
+
contiguous = tailStart <= headEnd;
|
|
426
|
+
tail = contiguous ? await readSlice(handle, headEnd, size - headEnd) : await readSlice(handle, tailStart, size - tailStart);
|
|
403
427
|
}
|
|
404
|
-
const tailStart = Math.max(0, size - TAIL_BYTES);
|
|
405
|
-
const contiguous = tailStart <= headEnd;
|
|
406
|
-
const tail = contiguous ? await readSlice(handle, headEnd, size - headEnd) : await readSlice(handle, tailStart, size - tailStart);
|
|
407
428
|
let cwd = "";
|
|
408
429
|
let createdAt = "";
|
|
409
430
|
let lastEvent = "";
|
|
@@ -438,9 +459,9 @@ async function readSession(file, now) {
|
|
|
438
459
|
parseLines(tail.slice(tail.indexOf("\n") + 1), accept);
|
|
439
460
|
}
|
|
440
461
|
if (!hasSessionMeta) throw new Error("Session metadata is missing or invalid.");
|
|
441
|
-
const updatedAt = new Date(
|
|
462
|
+
const updatedAt = new Date(source.info.mtimeMs).toISOString();
|
|
442
463
|
const projectName = cwd ? basename(cwd) || cwd : "Unknown project";
|
|
443
|
-
const active = !file.archive && lastEvent === "task_started" && now -
|
|
464
|
+
const active = !file.archive && lastEvent === "task_started" && now - source.info.mtimeMs < ACTIVE_WINDOW_MS;
|
|
444
465
|
const activity = active ? "A turn started recently; live activity is unverified."
|
|
445
466
|
: lastEvent === "task_complete" ? "Latest recorded turn ended; task outcome is unknown."
|
|
446
467
|
: lastEvent === "turn_aborted" ? "Latest recorded turn stopped; task outcome is unknown."
|
|
@@ -450,7 +471,7 @@ async function readSession(file, now) {
|
|
|
450
471
|
summary: activity, status: active ? "working" : null, createdAt: createdAt || updatedAt, updatedAt,
|
|
451
472
|
updatedBy: "Local Codex log", project: { name: projectName, path: cwd, githubRepos: [] },
|
|
452
473
|
threadId: file.id, turnRef: null, turnId: null, lastResult: null, latestTurn: null,
|
|
453
|
-
observed: { archive: file.archive, lastTurnEvent: lastEvent || null, lastTurnEventAt: lastEventAt || null, recentFileActivity: !file.archive && now -
|
|
474
|
+
observed: { archive: file.archive, lastTurnEvent: lastEvent || null, lastTurnEventAt: lastEventAt || null, recentFileActivity: !file.archive && now - source.info.mtimeMs < ACTIVE_WINDOW_MS, userMessages, assistantMessages, sampledBytes: Math.min(size, headEnd + TAIL_BYTES), fileBytes: size },
|
|
454
475
|
};
|
|
455
476
|
} finally { await handle.close(); }
|
|
456
477
|
}
|
|
@@ -551,7 +572,7 @@ export class CodexSessionScanner {
|
|
|
551
572
|
const nextTasks = new Map(grouped.tasks.map((task) => [task.id, task]));
|
|
552
573
|
const signature = (tasks) => JSON.stringify([...tasks.values()].map((task) => [
|
|
553
574
|
task.id, task.title, task.project.path, task.project.id, task.project.name, task.updatedAt, task.status, task.summary, task.turnError,
|
|
554
|
-
task.observed.archive, task.observed.unread, task.observed.lastTurnEvent, task.observed.directChildCount, task.observed.latestTurnDurationMs,
|
|
575
|
+
task.observed.archive, task.observed.pinned, task.observed.unread, task.observed.lastTurnEvent, task.observed.directChildCount, task.observed.latestTurnDurationMs,
|
|
555
576
|
task.scheduled, task.nextRunAt, task.inputSource, task.turnId, task.promptExcerpt, task.replyExcerpt, task.annotations, task.promptImages, task.promptImage, task.replyImage, task.queuedPrompts, task.needsAnswer, task.pendingQuestions, task.pullRequests,
|
|
556
577
|
]));
|
|
557
578
|
if (this.stats?.mode !== "database" || signature(this.tasks) !== signature(nextTasks) || JSON.stringify(this.projects) !== JSON.stringify(grouped.projects)) this.revision += 1;
|
|
@@ -698,9 +719,7 @@ export class CodexSessionScanner {
|
|
|
698
719
|
try {
|
|
699
720
|
const path = this.rolloutPaths.get(id);
|
|
700
721
|
if (typeof path !== "string" || !isAbsolute(path)) return detail;
|
|
701
|
-
const
|
|
702
|
-
if (!info.isFile()) return detail;
|
|
703
|
-
const log = await readSession({ path, id, archive: task.observed.archive, mtimeMs: info.mtimeMs }, this.now());
|
|
722
|
+
const log = await readSession({ path, id, archive: task.observed.archive }, this.now());
|
|
704
723
|
return { ...detail, observed: { ...detail.observed,
|
|
705
724
|
userMessages: log.observed.userMessages,
|
|
706
725
|
assistantMessages: log.observed.assistantMessages,
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import { migrateCache } from "./next-cache.js";
|
|
2
|
-
import {
|
|
3
|
-
import { readdir, stat, readFile } from "node:fs/promises";
|
|
2
|
+
import { readdir, readFile } from "node:fs/promises";
|
|
4
3
|
import { join, basename } from "node:path";
|
|
5
4
|
import { createHash } from "node:crypto";
|
|
6
5
|
import { writeDurableAtomic } from "./state-store.js";
|
|
7
|
-
import
|
|
6
|
+
import { plainRolloutPath, rolloutMetadata, openRollout, rolloutChunks } from "./codex-rollout.js";
|
|
8
7
|
import { estimateCallCost } from "./codex-pricing.js";
|
|
9
8
|
|
|
10
9
|
const FIELDS = ["input_tokens", "cached_input_tokens", "cache_write_input_tokens", "output_tokens", "reasoning_output_tokens", "total_tokens"];
|
|
@@ -77,7 +76,7 @@ async function inventory(root, paths = []) {
|
|
|
77
76
|
catch (error) { if (error.code === "ENOENT") return paths; throw error; }
|
|
78
77
|
for (const entry of entries) {
|
|
79
78
|
if (entry.isDirectory()) await inventory(join(root, entry.name), paths);
|
|
80
|
-
else if (entry.isFile() && /\.jsonl(?:\.zst)?$/.test(entry.name)) paths.push(join(root, entry.name));
|
|
79
|
+
else if (entry.isFile() && /\.jsonl(?:\.zst)?$/.test(entry.name)) paths.push(plainRolloutPath(join(root, entry.name)));
|
|
81
80
|
}
|
|
82
81
|
return paths;
|
|
83
82
|
}
|
|
@@ -113,10 +112,10 @@ export class CodexSessionUsage {
|
|
|
113
112
|
const sources = [];
|
|
114
113
|
for (const session of sessions) {
|
|
115
114
|
const paths = new Set(this.paths.filter(path => basename(path).includes(`-${session.id}`)));
|
|
116
|
-
if (session.rollout_path) paths.add(session.rollout_path);
|
|
115
|
+
if (session.rollout_path) paths.add(plainRolloutPath(session.rollout_path));
|
|
117
116
|
const versions = [];
|
|
118
117
|
for (const path of paths) {
|
|
119
|
-
try { const info = await
|
|
118
|
+
try { const { info } = await rolloutMetadata(path); versions.push([path, info.dev, info.ino, info.size, info.mtimeMs]); }
|
|
120
119
|
catch { versions.push([path, "missing"]); }
|
|
121
120
|
}
|
|
122
121
|
sources.push([session.id, versions]);
|
|
@@ -133,12 +132,20 @@ export class CodexSessionUsage {
|
|
|
133
132
|
}
|
|
134
133
|
const sourceKeys = (items) => items.flatMap(([sessionId, versions]) => versions
|
|
135
134
|
.filter(version => version.length > 2)
|
|
136
|
-
.map(version => JSON.stringify([sessionId, basename(version[0])])));
|
|
135
|
+
.map(version => JSON.stringify([sessionId, basename(plainRolloutPath(version[0]))])));
|
|
137
136
|
const currentSources = new Set(sourceKeys(sources));
|
|
138
137
|
let knownSources = [];
|
|
139
138
|
try {
|
|
140
139
|
// Older cache entries already contain source metadata in their signature.
|
|
141
140
|
knownSources = saved?.knownSources ?? sourceKeys(JSON.parse(saved.sourceSignature)[2]);
|
|
141
|
+
knownSources = [...new Set(knownSources.map(key => {
|
|
142
|
+
const [sessionId, name] = JSON.parse(key);
|
|
143
|
+
return JSON.stringify([sessionId, plainRolloutPath(name)]);
|
|
144
|
+
}))];
|
|
145
|
+
if (saved?.recordSources) saved = { ...saved, recordSources: Object.fromEntries(Object.entries(saved.recordSources).map(([key, rows]) => {
|
|
146
|
+
const [sessionId, name] = JSON.parse(key);
|
|
147
|
+
return [JSON.stringify([sessionId, plainRolloutPath(name)]), rows];
|
|
148
|
+
})) };
|
|
142
149
|
if (!Array.isArray(knownSources) || knownSources.some(key => {
|
|
143
150
|
if (typeof key !== "string") return true;
|
|
144
151
|
const parts = JSON.parse(key);
|
|
@@ -153,7 +160,7 @@ export class CodexSessionUsage {
|
|
|
153
160
|
const recordSources = {};
|
|
154
161
|
for (const session of sessions) {
|
|
155
162
|
const paths = new Set(this.paths.filter(path => basename(path).includes(`-${session.id}`)));
|
|
156
|
-
if (session.rollout_path) paths.add(session.rollout_path);
|
|
163
|
+
if (session.rollout_path) paths.add(plainRolloutPath(session.rollout_path));
|
|
157
164
|
const records = [], versions = [];
|
|
158
165
|
let failed = inventoryFailed, latestFailed = false, timePartial = inventoryFailed;
|
|
159
166
|
if (!paths.size) failed = latestFailed = timePartial = true;
|
|
@@ -169,7 +176,7 @@ export class CodexSessionUsage {
|
|
|
169
176
|
const sourceKey = JSON.stringify([session.id, basename(path)]);
|
|
170
177
|
if (Array.isArray(saved?.recordSources?.[sourceKey]) && !missingSources.includes(sourceKey)) missingSources.push(sourceKey);
|
|
171
178
|
retryRead ||= availablePaths.has(path);
|
|
172
|
-
latestFailed ||= session.id !== id || path === session.rollout_path;
|
|
179
|
+
latestFailed ||= session.id !== id || path === plainRolloutPath(session.rollout_path ?? "");
|
|
173
180
|
versions.push([path, "missing"]);
|
|
174
181
|
}
|
|
175
182
|
}
|
|
@@ -244,38 +251,42 @@ export class CodexSessionUsage {
|
|
|
244
251
|
return result;
|
|
245
252
|
}
|
|
246
253
|
async readFile(path, expectedId) {
|
|
247
|
-
const
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
cached
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
pending = Buffer.concat([pending, chunk]);
|
|
258
|
-
let end;
|
|
259
|
-
while ((end = pending.indexOf(10)) !== -1) {
|
|
260
|
-
const line = pending.subarray(0, end);
|
|
261
|
-
consumed += end + 1;
|
|
262
|
-
if (!skipping) this.line(cached, line, expectedId);
|
|
263
|
-
skipping = false;
|
|
264
|
-
pending = pending.subarray(end + 1);
|
|
265
|
-
cached.offset = consumed;
|
|
254
|
+
const source = await openRollout(path);
|
|
255
|
+
try {
|
|
256
|
+
path = plainRolloutPath(path);
|
|
257
|
+
const { info } = source;
|
|
258
|
+
if (!info.isFile() || info.size === 0) throw new Error("Rollout is empty or not a file.");
|
|
259
|
+
let cached = this.files.get(path);
|
|
260
|
+
if (cached && cached.identity === `${info.dev}:${info.ino}` && cached.size === info.size && cached.mtime === info.mtimeMs) return this.sourceView(cached, expectedId);
|
|
261
|
+
const compressed = source.path.endsWith(".zst");
|
|
262
|
+
if (!cached || compressed || cached.identity !== `${info.dev}:${info.ino}` || info.size < cached.size || info.size === cached.size) {
|
|
263
|
+
cached = { identity: `${info.dev}:${info.ino}`, offset: 0, records: [], partial: false, turn: null, root: null, model: null, valid: false };
|
|
266
264
|
}
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
265
|
+
let pending = Buffer.alloc(0), consumed = cached.offset, skipping = false;
|
|
266
|
+
for await (const chunk of rolloutChunks(source, { start: cached.offset, end: info.size - 1, highWaterMark: 256 * 1024 })) {
|
|
267
|
+
pending = Buffer.concat([pending, chunk]);
|
|
268
|
+
let end;
|
|
269
|
+
while ((end = pending.indexOf(10)) !== -1) {
|
|
270
|
+
const line = pending.subarray(0, end);
|
|
271
|
+
consumed += end + 1;
|
|
272
|
+
if (!skipping) this.line(cached, line, expectedId);
|
|
273
|
+
skipping = false;
|
|
274
|
+
pending = pending.subarray(end + 1);
|
|
275
|
+
cached.offset = consumed;
|
|
276
|
+
}
|
|
277
|
+
if (pending.length > 2 * 1024 * 1024) {
|
|
278
|
+
// Image/message bodies can be huge; accounting records are small.
|
|
279
|
+
if (/"type"\s*:\s*"(?:turn_context|session_meta|token_count)"/.test(pending.subarray(0, 256).toString())) this.issue(cached);
|
|
280
|
+
consumed += pending.length; pending = Buffer.alloc(0); skipping = true;
|
|
281
|
+
}
|
|
271
282
|
}
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
283
|
+
cached.size = info.size; cached.mtime = info.mtimeMs;
|
|
284
|
+
cached.tail = pending; cached.skipping = skipping;
|
|
285
|
+
const view = this.sourceView(cached, expectedId);
|
|
286
|
+
if (!view.valid) throw new Error("Rollout belongs to a different chat.");
|
|
287
|
+
this.files.set(path, cached);
|
|
288
|
+
return view;
|
|
289
|
+
} finally { await source.handle.close(); }
|
|
279
290
|
}
|
|
280
291
|
sourceView(cached, expectedId) {
|
|
281
292
|
// EOF may contain a complete JSON record or a partially written line.
|
|
@@ -310,15 +321,6 @@ export class CodexSessionUsage {
|
|
|
310
321
|
}
|
|
311
322
|
}
|
|
312
323
|
|
|
313
|
-
async function* readChunks(path, options, compressed) {
|
|
314
|
-
if (compressed && typeof zlib.createZstdDecompress !== "function") throw new Error("This runtime cannot decode compressed rollouts.");
|
|
315
|
-
const input = createReadStream(path, options);
|
|
316
|
-
const stream = compressed ? zlib.createZstdDecompress() : input;
|
|
317
|
-
if (compressed) { input.on("error", error => stream.destroy(error)); input.pipe(stream); }
|
|
318
|
-
try { for await (const chunk of stream) yield chunk; }
|
|
319
|
-
finally { input.destroy(); if (compressed) stream.destroy(); }
|
|
320
|
-
}
|
|
321
|
-
|
|
322
324
|
function validResult(value) {
|
|
323
325
|
return value && ["latest", "total"].every(key => {
|
|
324
326
|
const part = value[key];
|