taskchef 9.0.3 → 9.0.4

package/.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "taskchef",
-  "version": "9.0.3",
+  "version": "9.0.4",
   "description": "Native Codex dashboard with lightweight project delegation.",
   "author": {
     "name": "Favo Yang",

package/docs/taskchef-next-storage.md

@@ -88,6 +88,16 @@ not read again. Appended files are read from the last complete line; replaced
 or truncated files are read again. Changed compressed files are decoded again;
 unchanged compressed files use the same cache. A runtime without zstd support
 marks compressed histories partial. Concurrent views share this reader.
+Plain and compressed siblings share one cache identity. Reads prefer the plain
+file when both exist and retry briefly during format transitions. A stale
+selected path can resolve its sibling, so compression or restoration alone does
+not mark accounting partial. Existing compressed cache identities are normalized
+when loaded.
+
+Optional Details message counts use the same path resolution. Compressed logs
+are decoded as a stream, retaining bounded head and tail samples. Their
+`fileBytes` and `sampledBytes` describe decoded JSONL bytes, as for plain files.
+Compression does not change database-derived board status or replies.
 
 Computed accounting results are saved under TaskChef's state directory in
 `cache/usage/`, with one hashed file per chat and duration-view variant. Files

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "taskchef",
-  "version": "9.0.3",
+  "version": "9.0.4",
   "description": "A native Codex dashboard for chats, usage, pull requests, and schedules.",
   "license": "Apache-2.0",
   "author": "Favo Yang",

package/src/codex-rollout.js

@@ -0,0 +1,36 @@
+import { open, stat } from "node:fs/promises";
+import * as zlib from "node:zlib";
+
+export function plainRolloutPath(path) { return path.replace(/\.jsonl\.zst$/, ".jsonl"); }
+
+// Codex can replace either representation while readers are resolving a path.
+async function resolve(path, inspect) {
+  const plain = plainRolloutPath(path);
+  for (let attempt = 0; ; attempt++) {
+    for (const candidate of [plain, `${plain}.zst`]) {
+      try { return { path: candidate, ...await inspect(candidate) }; }
+      catch (error) { if (error.code !== "ENOENT") throw error; }
+    }
+    if (attempt === 3) { const error = new Error("Rollout is missing."); error.code = "ENOENT"; throw error; }
+    await new Promise(resolve => setTimeout(resolve, 50));
+  }
+}
+
+export function rolloutMetadata(path) { return resolve(path, async path => ({ info: await stat(path) })); }
+export function openRollout(path) {
+  return resolve(path, async path => {
+    const handle = await open(path, "r");
+    try { return { handle, info: await handle.stat() }; }
+    catch (error) { await handle.close(); throw error; }
+  });
+}
+
+export async function* rolloutChunks({ path, handle }, options = {}) {
+  const compressed = path.endsWith(".jsonl.zst");
+  if (compressed && typeof zlib.createZstdDecompress !== "function") throw new Error("This runtime cannot decode compressed rollouts.");
+  const input = handle.createReadStream({ ...options, autoClose: false });
+  const stream = compressed ? zlib.createZstdDecompress() : input;
+  if (compressed) { input.on("error", error => stream.destroy(error)); input.pipe(stream); }
+  try { for await (const chunk of stream) yield chunk; }
+  finally { input.destroy(); if (compressed) stream.destroy(); }
+}

package/src/codex-session-scanner.js

@@ -1,3 +1,4 @@
+import { openRollout, rolloutChunks } from "./codex-rollout.js";
 import { createHash } from "node:crypto";
 import { normalizeUri } from "micromark-util-sanitize-uri";
 import { displayUserPrompt, responseAnnotations } from "./user-prompt.js";
@@ -7,7 +8,7 @@ import { CodexPricing } from "./codex-pricing.js";
 import { replyImage, replyImages, localReplyImage } from "./reply-image.js";
 import { homedir } from "node:os";
 import { basename, isAbsolute, join } from "node:path";
-import { open, stat, readFile, readdir, mkdir } from "node:fs/promises";
+import { stat, readFile, readdir, mkdir } from "node:fs/promises";
 import { acquireFileLock } from "./file-lock.js";
 import { writeDurableAtomic } from "./state-store.js";
 import { dirname } from "node:path";
@@ -392,18 +393,37 @@ async function readSlice(handle, start, bytes) {
 }
 
 async function readSession(file, now) {
-  const handle = await open(file.path, "r");
+  const source = await openRollout(file.path);
+  const { handle } = source;
   try {
-    const size = (await handle.stat()).size;
-    let headEnd = Math.min(size, HEAD_BYTES);
-    let head = await readSlice(handle, 0, headEnd);
-    if (!head.includes("\n") && headEnd < size) {
-      headEnd = Math.min(size, META_LINE_BYTES);
+    let size, headEnd, head, tail, contiguous;
+    if (source.path.endsWith(".jsonl.zst")) {
+      let prefix = Buffer.alloc(0), suffix = Buffer.alloc(0);
+      size = 0;
+      for await (const chunk of rolloutChunks(source, { highWaterMark: TAIL_BYTES })) {
+        size += chunk.length;
+        if (prefix.length < META_LINE_BYTES) prefix = Buffer.concat([prefix, chunk.subarray(0, META_LINE_BYTES - prefix.length)]);
+        suffix = Buffer.concat([suffix, chunk]);
+        if (suffix.length > TAIL_BYTES) suffix = Buffer.from(suffix.subarray(suffix.length - TAIL_BYTES));
+      }
+      headEnd = Math.min(size, HEAD_BYTES);
+      if (!prefix.subarray(0, headEnd).includes(10) && headEnd < size) headEnd = Math.min(size, META_LINE_BYTES);
+      head = prefix.subarray(0, headEnd).toString("utf8");
+      const tailStart = Math.max(0, size - TAIL_BYTES);
+      contiguous = tailStart <= headEnd;
+      tail = suffix.subarray(contiguous ? headEnd - tailStart : 0).toString("utf8");
+    } else {
+      size = source.info.size;
+      headEnd = Math.min(size, HEAD_BYTES);
       head = await readSlice(handle, 0, headEnd);
+      if (!head.includes("\n") && headEnd < size) {
+        headEnd = Math.min(size, META_LINE_BYTES);
+        head = await readSlice(handle, 0, headEnd);
+      }
+      const tailStart = Math.max(0, size - TAIL_BYTES);
+      contiguous = tailStart <= headEnd;
+      tail = contiguous ? await readSlice(handle, headEnd, size - headEnd) : await readSlice(handle, tailStart, size - tailStart);
     }
-    const tailStart = Math.max(0, size - TAIL_BYTES);
-    const contiguous = tailStart <= headEnd;
-    const tail = contiguous ? await readSlice(handle, headEnd, size - headEnd) : await readSlice(handle, tailStart, size - tailStart);
     let cwd = "";
     let createdAt = "";
     let lastEvent = "";
@@ -438,9 +458,9 @@ async function readSession(file, now) {
       parseLines(tail.slice(tail.indexOf("\n") + 1), accept);
     }
     if (!hasSessionMeta) throw new Error("Session metadata is missing or invalid.");
-    const updatedAt = new Date(file.mtimeMs).toISOString();
+    const updatedAt = new Date(source.info.mtimeMs).toISOString();
     const projectName = cwd ? basename(cwd) || cwd : "Unknown project";
-    const active = !file.archive && lastEvent === "task_started" && now - file.mtimeMs < ACTIVE_WINDOW_MS;
+    const active = !file.archive && lastEvent === "task_started" && now - source.info.mtimeMs < ACTIVE_WINDOW_MS;
     const activity = active ? "A turn started recently; live activity is unverified."
       : lastEvent === "task_complete" ? "Latest recorded turn ended; task outcome is unknown."
         : lastEvent === "turn_aborted" ? "Latest recorded turn stopped; task outcome is unknown."
@@ -450,7 +470,7 @@ async function readSession(file, now) {
       summary: activity, status: active ? "working" : null, createdAt: createdAt || updatedAt, updatedAt,
       updatedBy: "Local Codex log", project: { name: projectName, path: cwd, githubRepos: [] },
       threadId: file.id, turnRef: null, turnId: null, lastResult: null, latestTurn: null,
-      observed: { archive: file.archive, lastTurnEvent: lastEvent || null, lastTurnEventAt: lastEventAt || null, recentFileActivity: !file.archive && now - file.mtimeMs < ACTIVE_WINDOW_MS, userMessages, assistantMessages, sampledBytes: Math.min(size, headEnd + TAIL_BYTES), fileBytes: size },
+      observed: { archive: file.archive, lastTurnEvent: lastEvent || null, lastTurnEventAt: lastEventAt || null, recentFileActivity: !file.archive && now - source.info.mtimeMs < ACTIVE_WINDOW_MS, userMessages, assistantMessages, sampledBytes: Math.min(size, headEnd + TAIL_BYTES), fileBytes: size },
     };
   } finally { await handle.close(); }
 }
@@ -698,9 +718,7 @@ export class CodexSessionScanner {
     try {
       const path = this.rolloutPaths.get(id);
       if (typeof path !== "string" || !isAbsolute(path)) return detail;
-      const info = await stat(path);
-      if (!info.isFile()) return detail;
-      const log = await readSession({ path, id, archive: task.observed.archive, mtimeMs: info.mtimeMs }, this.now());
+      const log = await readSession({ path, id, archive: task.observed.archive }, this.now());
       return { ...detail, observed: { ...detail.observed,
         userMessages: log.observed.userMessages,
         assistantMessages: log.observed.assistantMessages,

package/src/codex-session-usage.js

@@ -1,10 +1,9 @@
 import { migrateCache } from "./next-cache.js";
-import { createReadStream } from "node:fs";
-import { readdir, stat, readFile } from "node:fs/promises";
+import { readdir, readFile } from "node:fs/promises";
 import { join, basename } from "node:path";
 import { createHash } from "node:crypto";
 import { writeDurableAtomic } from "./state-store.js";
-import * as zlib from "node:zlib";
+import { plainRolloutPath, rolloutMetadata, openRollout, rolloutChunks } from "./codex-rollout.js";
 import { estimateCallCost } from "./codex-pricing.js";
 
 const FIELDS = ["input_tokens", "cached_input_tokens", "cache_write_input_tokens", "output_tokens", "reasoning_output_tokens", "total_tokens"];
@@ -77,7 +76,7 @@ async function inventory(root, paths = []) {
   catch (error) { if (error.code === "ENOENT") return paths; throw error; }
   for (const entry of entries) {
     if (entry.isDirectory()) await inventory(join(root, entry.name), paths);
-    else if (entry.isFile() && /\.jsonl(?:\.zst)?$/.test(entry.name)) paths.push(join(root, entry.name));
+    else if (entry.isFile() && /\.jsonl(?:\.zst)?$/.test(entry.name)) paths.push(plainRolloutPath(join(root, entry.name)));
   }
   return paths;
 }
@@ -113,10 +112,10 @@ export class CodexSessionUsage {
     const sources = [];
     for (const session of sessions) {
       const paths = new Set(this.paths.filter(path => basename(path).includes(`-${session.id}`)));
-      if (session.rollout_path) paths.add(session.rollout_path);
+      if (session.rollout_path) paths.add(plainRolloutPath(session.rollout_path));
       const versions = [];
       for (const path of paths) {
-        try { const info = await stat(path); versions.push([path, info.dev, info.ino, info.size, info.mtimeMs]); }
+        try { const { info } = await rolloutMetadata(path); versions.push([path, info.dev, info.ino, info.size, info.mtimeMs]); }
         catch { versions.push([path, "missing"]); }
       }
       sources.push([session.id, versions]);
@@ -133,12 +132,20 @@ export class CodexSessionUsage {
     }
     const sourceKeys = (items) => items.flatMap(([sessionId, versions]) => versions
       .filter(version => version.length > 2)
-      .map(version => JSON.stringify([sessionId, basename(version[0])])));
+      .map(version => JSON.stringify([sessionId, basename(plainRolloutPath(version[0]))])));
     const currentSources = new Set(sourceKeys(sources));
     let knownSources = [];
     try {
       // Older cache entries already contain source metadata in their signature.
       knownSources = saved?.knownSources ?? sourceKeys(JSON.parse(saved.sourceSignature)[2]);
+      knownSources = [...new Set(knownSources.map(key => {
+        const [sessionId, name] = JSON.parse(key);
+        return JSON.stringify([sessionId, plainRolloutPath(name)]);
+      }))];
+      if (saved?.recordSources) saved = { ...saved, recordSources: Object.fromEntries(Object.entries(saved.recordSources).map(([key, rows]) => {
+        const [sessionId, name] = JSON.parse(key);
+        return [JSON.stringify([sessionId, plainRolloutPath(name)]), rows];
+      })) };
       if (!Array.isArray(knownSources) || knownSources.some(key => {
         if (typeof key !== "string") return true;
         const parts = JSON.parse(key);
@@ -153,7 +160,7 @@ export class CodexSessionUsage {
     const recordSources = {};
     for (const session of sessions) {
       const paths = new Set(this.paths.filter(path => basename(path).includes(`-${session.id}`)));
-      if (session.rollout_path) paths.add(session.rollout_path);
+      if (session.rollout_path) paths.add(plainRolloutPath(session.rollout_path));
       const records = [], versions = [];
       let failed = inventoryFailed, latestFailed = false, timePartial = inventoryFailed;
       if (!paths.size) failed = latestFailed = timePartial = true;
@@ -169,7 +176,7 @@ export class CodexSessionUsage {
           const sourceKey = JSON.stringify([session.id, basename(path)]);
           if (Array.isArray(saved?.recordSources?.[sourceKey]) && !missingSources.includes(sourceKey)) missingSources.push(sourceKey);
           retryRead ||= availablePaths.has(path);
-          latestFailed ||= session.id !== id || path === session.rollout_path;
+          latestFailed ||= session.id !== id || path === plainRolloutPath(session.rollout_path ?? "");
           versions.push([path, "missing"]);
         }
       }
@@ -244,38 +251,42 @@ export class CodexSessionUsage {
     return result;
   }
   async readFile(path, expectedId) {
-    const info = await stat(path);
-    if (!info.isFile() || info.size === 0) throw new Error("Rollout is empty or not a file.");
-    let cached = this.files.get(path);
-    if (cached && cached.identity === `${info.dev}:${info.ino}` && cached.size === info.size && cached.mtime === info.mtimeMs) return this.sourceView(cached, expectedId);
-    const compressed = path.endsWith(".zst");
-    if (!cached || compressed || cached.identity !== `${info.dev}:${info.ino}` || info.size < cached.size || info.size === cached.size) {
-      cached = { identity: `${info.dev}:${info.ino}`, offset: 0, records: [], partial: false, turn: null, root: null, model: null, valid: false };
-    }
-    let pending = Buffer.alloc(0), consumed = cached.offset, skipping = false;
-    for await (const chunk of readChunks(path, { start: cached.offset, end: info.size - 1, highWaterMark: 256 * 1024 }, compressed)) {
-      pending = Buffer.concat([pending, chunk]);
-      let end;
-      while ((end = pending.indexOf(10)) !== -1) {
-        const line = pending.subarray(0, end);
-        consumed += end + 1;
-        if (!skipping) this.line(cached, line, expectedId);
-        skipping = false;
-        pending = pending.subarray(end + 1);
-        cached.offset = consumed;
+    const source = await openRollout(path);
+    try {
+      path = plainRolloutPath(path);
+      const { info } = source;
+      if (!info.isFile() || info.size === 0) throw new Error("Rollout is empty or not a file.");
+      let cached = this.files.get(path);
+      if (cached && cached.identity === `${info.dev}:${info.ino}` && cached.size === info.size && cached.mtime === info.mtimeMs) return this.sourceView(cached, expectedId);
+      const compressed = source.path.endsWith(".zst");
+      if (!cached || compressed || cached.identity !== `${info.dev}:${info.ino}` || info.size < cached.size || info.size === cached.size) {
+        cached = { identity: `${info.dev}:${info.ino}`, offset: 0, records: [], partial: false, turn: null, root: null, model: null, valid: false };
       }
-      if (pending.length > 2 * 1024 * 1024) {
-        // Image/message bodies can be huge; accounting records are small.
-        if (/"type"\s*:\s*"(?:turn_context|session_meta|token_count)"/.test(pending.subarray(0, 256).toString())) this.issue(cached);
-        consumed += pending.length; pending = Buffer.alloc(0); skipping = true;
+      let pending = Buffer.alloc(0), consumed = cached.offset, skipping = false;
+      for await (const chunk of rolloutChunks(source, { start: cached.offset, end: info.size - 1, highWaterMark: 256 * 1024 })) {
+        pending = Buffer.concat([pending, chunk]);
+        let end;
+        while ((end = pending.indexOf(10)) !== -1) {
+          const line = pending.subarray(0, end);
+          consumed += end + 1;
+          if (!skipping) this.line(cached, line, expectedId);
+          skipping = false;
+          pending = pending.subarray(end + 1);
+          cached.offset = consumed;
+        }
+        if (pending.length > 2 * 1024 * 1024) {
+          // Image/message bodies can be huge; accounting records are small.
+          if (/"type"\s*:\s*"(?:turn_context|session_meta|token_count)"/.test(pending.subarray(0, 256).toString())) this.issue(cached);
+          consumed += pending.length; pending = Buffer.alloc(0); skipping = true;
+        }
       }
-    }
-    cached.size = info.size; cached.mtime = info.mtimeMs;
-    cached.tail = pending; cached.skipping = skipping;
-    const view = this.sourceView(cached, expectedId);
-    if (!view.valid) throw new Error("Rollout belongs to a different chat.");
-    this.files.set(path, cached);
-    return view;
+      cached.size = info.size; cached.mtime = info.mtimeMs;
+      cached.tail = pending; cached.skipping = skipping;
+      const view = this.sourceView(cached, expectedId);
+      if (!view.valid) throw new Error("Rollout belongs to a different chat.");
+      this.files.set(path, cached);
+      return view;
+    } finally { await source.handle.close(); }
   }
   sourceView(cached, expectedId) {
     // EOF may contain a complete JSON record or a partially written line.
@@ -310,15 +321,6 @@ export class CodexSessionUsage {
   }
 }
 
-async function* readChunks(path, options, compressed) {
-  if (compressed && typeof zlib.createZstdDecompress !== "function") throw new Error("This runtime cannot decode compressed rollouts.");
-  const input = createReadStream(path, options);
-  const stream = compressed ? zlib.createZstdDecompress() : input;
-  if (compressed) { input.on("error", error => stream.destroy(error)); input.pipe(stream); }
-  try { for await (const chunk of stream) yield chunk; }
-  finally { input.destroy(); if (compressed) stream.destroy(); }
-}
-
 function validResult(value) {
   return value && ["latest", "total"].every(key => {
     const part = value[key];