sidekick-shared 0.27.1 → 0.27.3

package/dist/promptHistory.js

@@ -9,9 +9,15 @@
  * or somewhere inside one. Every bound stops cleanly and is reported in
  * `boundsHit`; nothing is silently truncated and no bound throws.
  *
+ * Files are streamed in fixed-size chunks and every pass stops at a complete
+ * line, so a large or busy log is read across several calls: the cursor holds
+ * where each file stopped and the next call continues from there. A single
+ * record over `maxRecordBytes` is skipped without being buffered and reported
+ * in `exclusions`, the only way a record is permanently left out.
+ *
  * A prompt can be written before its answer. The cursor then keeps the byte
- * offset of that prompt's line (not its text), the next call replays from it,
- * and the prompt is returned again under the same identity once its first
+ * offset of that prompt's line and its answer-tracking state (never text), and
+ * the prompt is returned again under the same identity once its first
  * answering call brings model or usage (`metadataStatus` says how settled).
  */
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
@@ -48,6 +54,7 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.__promptHistoryTesting = void 0;
 exports.collectPromptHistory = collectPromptHistory;
 const crypto_1 = require("crypto");
 const fs = __importStar(require("fs"));
@@ -60,10 +67,22 @@ const DEFAULT_BOUNDS = {
     maxSessions: 1000,
     maxFileBytes: 16 * 1024 * 1024,
     maxTotalBytes: 256 * 1024 * 1024,
+    maxRecordBytes: 16 * 1024 * 1024,
+    maxEntries: 10_000,
     deadlineMs: 30_000,
 };
+const DEFAULT_READ_CHUNK_BYTES = 1024 * 1024;
+const HEAD_BYTES = 4096;
+const MAX_EXCLUSIONS = 1000;
 const CHECK_EVERY_LINES = 500;
 const ALL_PROVIDERS = ['claude-code', 'codex'];
+let readChunkBytes = DEFAULT_READ_CHUNK_BYTES;
+/** Test hooks; not part of the public entry points. */
+exports.__promptHistoryTesting = {
+    setReadChunkBytes(bytes) {
+        readChunkBytes = bytes === null ? DEFAULT_READ_CHUNK_BYTES : Math.max(1, Math.floor(bytes));
+    },
+};
 /** Collect human-typed prompts from Claude Code and Codex logs under the given roots. */
 async function collectPromptHistory(options) {
     const bounds = { ...DEFAULT_BOUNDS, ...definedBounds(options.bounds) };
@@ -74,13 +93,19 @@ async function collectPromptHistory(options) {
         sessionsSkippedUnchanged: 0,
         sessionsOutOfScope: 0,
         filesOverSizeLimit: 0,
+        sessionsWithBacklog: 0,
+        sessionsRewritten: 0,
         sessionsNotInteractive: 0,
+        recordsOverSizeLimit: 0,
+        recordsMalformed: 0,
         promptsMissingTimestamp: 0,
         promptsOutOfScope: 0,
     };
-    const previous = acceptedCursorSessions(options.cursor);
-    const cursor = { version: 2, sessions: { ...previous } };
+    const { sessions: previous, next: previousNext } = acceptedCursor(options.cursor);
+    const cursor = { version: 3, sessions: { ...previous } };
     const entries = [];
+    const exclusions = [];
+    let hasMore = false;
     const shouldStop = () => {
         if (options.signal?.aborted)
             return 'aborted';
@@ -93,6 +118,8 @@ async function collectPromptHistory(options) {
         cursor,
         stats,
         boundsHit: [...boundsHit],
+        exclusions,
+        hasMore,
     });
     const realpaths = new Map();
     const { roots, listingRoots } = resolveRoots(options.workspacePaths, options.includeWorktrees !== false, realpaths);
@@ -100,13 +127,36 @@ async function collectPromptHistory(options) {
         return finish();
     const sinceMs = options.since ? options.since.getTime() : null;
     const providers = options.providers ?? ALL_PROVIDERS;
-    const candidates = await listCandidates(providers, listingRoots, sinceMs);
-    const context = { roots, sinceMs, stats, realpaths, shouldStop };
+    const candidates = rotate(await listCandidates(providers, listingRoots, sinceMs), previousNext);
+    const context = {
+        roots,
+        sinceMs,
+        stats,
+        realpaths,
+        bounds,
+        shouldStop,
+        entriesSoFar: () => entries.length,
+        exclude: (exclusion) => {
+            if (exclusions.length < MAX_EXCLUSIONS)
+                exclusions.push(exclusion);
+        },
+    };
     let totalBytes = 0;
+    let scannedAny = false;
+    // A call-wide stop remembers where it stopped so the next call starts there.
+    const stopAt = (candidate, bound) => {
+        boundsHit.add(bound);
+        cursor.next = candidate.key;
+        hasMore = true;
+    };
     for (const candidate of candidates) {
         const stop = shouldStop();
         if (stop) {
-            boundsHit.add(stop);
+            stopAt(candidate, stop);
+            break;
+        }
+        if (scannedAny && entries.length >= bounds.maxEntries) {
+            stopAt(candidate, 'maxEntries');
             break;
         }
         let stat;
@@ -116,81 +166,67 @@ async function collectPromptHistory(options) {
         catch {
             continue;
         }
-        const prior = previous[candidate.key];
-        if (prior && prior.size === stat.size && prior.mtimeMs === stat.mtimeMs) {
+        const prior = validSession(previous[candidate.key]);
+        if (prior && isSettled(prior) && prior.size === stat.size && prior.mtimeMs === stat.mtimeMs) {
             stats.sessionsSkippedUnchanged++;
             continue;
         }
-        // A file that shrank was rewritten; its old offsets and ordinals are meaningless.
-        const resume = prior && prior.offset <= stat.size ? prior : undefined;
-        // Replay from the open prompt's line so late answer metadata can reach it.
-        let replay = validPending(resume);
-        let offset = replay?.offset ?? resume?.offset ?? 0;
-        if (resume?.state?.excluded) {
-            cursor.sessions[candidate.key] = {
-                ...resume,
-                size: stat.size,
-                mtimeMs: stat.mtimeMs,
-                offset: stat.size,
-            };
-            stats.sessionsNotInteractive++;
-            continue;
-        }
-        if (replay && resume && stat.size - offset > bounds.maxFileBytes) {
-            // Too far to replay: stop waiting on that prompt rather than stall the file.
-            replay = undefined;
-            offset = resume.offset;
-        }
-        const bytesToRead = stat.size - offset;
-        if (bytesToRead > bounds.maxFileBytes) {
-            stats.filesOverSizeLimit++;
-            boundsHit.add('maxFileBytes');
-            continue;
-        }
         if (stats.sessionsScanned >= bounds.maxSessions) {
-            boundsHit.add('maxSessions');
+            stopAt(candidate, 'maxSessions');
             break;
         }
-        if (totalBytes + bytesToRead > bounds.maxTotalBytes) {
-            boundsHit.add('maxTotalBytes');
+        const remainingTotal = bounds.maxTotalBytes - totalBytes;
+        if (scannedAny && remainingTotal <= 0) {
+            stopAt(candidate, 'maxTotalBytes');
             break;
         }
-        let buffer;
+        const budget = bounds.maxFileBytes <= remainingTotal
+            ? { bytes: bounds.maxFileBytes, bound: 'maxFileBytes', forceProgress: !scannedAny }
+            : {
+                bytes: Math.max(0, remainingTotal),
+                bound: 'maxTotalBytes',
+                forceProgress: !scannedAny,
+            };
+        stats.sessionsScanned++;
+        scannedAny = true;
+        let outcome;
         try {
-            buffer = await readRange(candidate.filePath, offset, bytesToRead);
+            outcome = await scanFile(candidate, stat, prior, budget, context);
         }
         catch {
             continue;
         }
-        totalBytes += buffer.length;
-        stats.sessionsScanned++;
-        const outcome = await scanBuffer(candidate, buffer, offset, { resume, replay }, context);
-        if (outcome.stoppedBy) {
-            // Partially processed file: drop its entries and leave its cursor untouched.
-            boundsHit.add(outcome.stoppedBy);
-            break;
-        }
+        totalBytes += outcome.bytesRead;
+        entries.push(...outcome.entries);
         if (outcome.notInteractive)
             stats.sessionsNotInteractive++;
         else if (outcome.inScope === 0 && outcome.outOfScope > 0)
             stats.sessionsOutOfScope++;
-        for (const entry of outcome.entries) {
-            // A replayed prompt is returned again only when its metadata changed.
-            if (replay && entry.ordinal === replay.ordinal && metadataDigest(entry) === replay.emitted) {
-                continue;
-            }
-            entries.push(entry);
+        if (outcome.session) {
+            cursor.sessions[candidate.key] = outcome.session;
+            if (outcome.session.backlog)
+                stats.sessionsWithBacklog++;
+        }
+        else {
+            // Rewritten under us: forget the file so the next call reads it from the start.
+            delete cursor.sessions[candidate.key];
+            stats.sessionsRewritten++;
+            hasMore = true;
+        }
+        if (outcome.stoppedBy === 'maxFileBytes' && outcome.progressed) {
+            // Per-file budget: move on so one large file cannot starve the others.
+            boundsHit.add('maxFileBytes');
+            stats.filesOverSizeLimit++;
+            hasMore = true;
+        }
+        else if (outcome.stoppedBy) {
+            // A file that could not move within its budget starts the next call, where
+            // the first file may always finish its first record.
+            if (outcome.stoppedBy === 'maxFileBytes')
+                stats.filesOverSizeLimit++;
+            stopAt(candidate, outcome.stoppedBy);
+            break;
         }
-        cursor.sessions[candidate.key] = {
-            provider: candidate.provider,
-            sessionId: candidate.sessionId,
-            size: stat.size,
-            mtimeMs: stat.mtimeMs,
-            lastOrdinal: outcome.lastOrdinal,
-            offset: outcome.offset,
-            ...(outcome.state ? { state: outcome.state } : {}),
-            ...(outcome.pending ? { pending: outcome.pending } : {}),
-        };
         await yieldToEventLoop();
     }
     return finish();
@@ -206,25 +242,64 @@ function definedBounds(bounds) {
     }
     return result;
 }
-function acceptedCursorSessions(cursor) {
-    if (!cursor || (cursor.version !== 1 && cursor.version !== 2))
-        return {};
+function acceptedCursor(cursor) {
+    if (!cursor || ![1, 2, 3].includes(cursor.version))
+        return { sessions: {} };
     const sessions = cursor.sessions;
-    return typeof sessions === 'object' && sessions !== null
-        ? sessions
-        : {};
+    return {
+        sessions: typeof sessions === 'object' && sessions !== null
+            ? sessions
+            : {},
+        ...(typeof cursor.next === 'string' ? { next: cursor.next } : {}),
+    };
+}
+function isOffset(value) {
+    return typeof value === 'number' && Number.isInteger(value) && value >= 0;
 }
-function validPending(resume) {
-    const pending = resume?.pending;
-    if (!resume || !pending)
+function validSession(session) {
+    if (!session || typeof session !== 'object')
         return undefined;
-    const valid = Number.isInteger(pending.ordinal) &&
-        pending.ordinal === resume.lastOrdinal &&
-        Number.isInteger(pending.offset) &&
-        pending.offset >= 0 &&
-        pending.offset <= resume.offset &&
+    const valid = isOffset(session.offset) &&
+        Number.isInteger(session.lastOrdinal) &&
+        session.lastOrdinal >= -1 &&
+        typeof session.size === 'number' &&
+        typeof session.mtimeMs === 'number';
+    return valid ? session : undefined;
+}
+/** A session whose last pass reached the end and needs no migration. */
+function isSettled(session) {
+    if (session.backlog || session.skipping)
+        return false;
+    if (session.state?.excluded)
+        return true;
+    if (!session.head)
+        return false;
+    return !session.pending || session.pending.answer !== undefined;
+}
+function resumePlan(session) {
+    if (!session)
+        return { kind: 'fresh' };
+    const pending = session.pending;
+    if (!pending)
+        return { kind: 'continue', session };
+    const basic = Number.isInteger(pending.ordinal) &&
+        pending.ordinal === session.lastOrdinal &&
+        isOffset(pending.offset) &&
+        pending.offset <= session.offset &&
         typeof pending.emitted === 'string';
-    return valid ? pending : undefined;
+    if (!basic)
+        return { kind: 'continue', session };
+    const answer = pending.answer;
+    if (answer === undefined) {
+        return session.skipping ? { kind: 'continue', session } : { kind: 'replay', session, pending };
+    }
+    const carried = typeof answer === 'object' &&
+        answer !== null &&
+        (answer.status === 'pending' || answer.status === 'provisional') &&
+        isOffset(pending.lineBytes) &&
+        pending.offset + pending.lineBytes < session.offset &&
+        typeof pending.timestamp === 'string';
+    return carried ? { kind: 'continue', session, carried: pending } : { kind: 'continue', session };
 }
 /** Digest of what a consumer would store for an entry besides its identity and text. */
 function metadataDigest(entry) {
@@ -249,6 +324,10 @@ function compareEntries(left, right) {
 function compareStrings(left, right) {
     return left < right ? -1 : left > right ? 1 : 0;
 }
+function rotate(candidates, nextKey) {
+    const index = nextKey ? candidates.findIndex((candidate) => candidate.key === nextKey) : -1;
+    return index <= 0 ? candidates : [...candidates.slice(index), ...candidates.slice(0, index)];
+}
 function realpathOrNull(input, cache) {
     const cached = cache.get(input);
     if (cached !== undefined)
@@ -343,69 +422,313 @@ async function listCandidates(providers, listingRoots, sinceMs) {
 function hashPath(filePath) {
     return (0, crypto_1.createHash)('sha256').update(filePath).digest('hex').slice(0, 12);
 }
-async function readRange(filePath, offset, length) {
+function hashBytes(buffer) {
+    return (0, crypto_1.createHash)('sha256').update(buffer).digest('hex').slice(0, 32);
+}
+async function readAt(handle, offset, length) {
     const buffer = Buffer.alloc(length);
-    if (length === 0)
-        return buffer;
-    const handle = await fs.promises.open(filePath, 'r');
-    try {
-        let read = 0;
-        while (read < length) {
-            const { bytesRead } = await handle.read(buffer, read, length - read, offset + read);
-            if (bytesRead === 0)
-                break;
-            read += bytesRead;
-        }
-        return read === length ? buffer : buffer.subarray(0, read);
-    }
-    finally {
-        await handle.close();
+    let read = 0;
+    while (read < length) {
+        const { bytesRead } = await handle.read(buffer, read, length - read, offset + read);
+        if (bytesRead === 0)
+            break;
+        read += bytesRead;
     }
+    return read === length ? buffer : buffer.subarray(0, read);
 }
 function yieldToEventLoop() {
     return new Promise((resolve) => setImmediate(resolve));
 }
-async function scanBuffer(candidate, buffer, startOffset, start, context) {
-    const lastNewline = buffer.lastIndexOf(0x0a);
-    const completeLength = lastNewline + 1;
-    const scanner = candidate.provider === 'claude-code'
-        ? new ClaudeLineScanner(candidate, start, context)
-        : new CodexLineScanner(candidate, start, context);
-    let lineStart = 0;
-    for (let index = 0; lineStart < completeLength; index++) {
-        if (index > 0 && index % CHECK_EVERY_LINES === 0) {
-            await yieldToEventLoop();
-            const stop = context.shouldStop();
-            if (stop)
-                return { ...scanner.outcome(startOffset), stoppedBy: stop };
-        }
-        const lineEnd = buffer.indexOf(0x0a, lineStart);
-        const line = buffer.toString('utf8', lineStart, lineEnd);
-        const lineOffset = startOffset + lineStart;
-        lineStart = lineEnd + 1;
-        if (!line.trim())
-            continue;
-        let record;
-        try {
-            record = JSON.parse(line);
+/**
+ * Checks that the prefix and resume point recorded for a file still hold.
+ * Returns the head to store, or null when the file was truncated or rewritten.
+ */
+async function verifyFile(handle, size, session) {
+    const headBytes = Math.min(HEAD_BYTES, size);
+    const head = await readAt(handle, 0, headBytes);
+    let bytesRead = head.length;
+    if (head.length !== headBytes)
+        return null;
+    if (session) {
+        if (session.offset > size)
+            return null;
+        const known = session.head;
+        if (known) {
+            if (!isOffset(known.bytes) || known.bytes > head.length)
+                return null;
+            if (hashBytes(head.subarray(0, known.bytes)) !== known.hash)
+                return null;
         }
-        catch {
-            continue;
+        if (session.offset > 0 && !session.skipping) {
+            const before = await readAt(handle, session.offset - 1, 1);
+            bytesRead += before.length;
+            if (before.length !== 1 || before[0] !== 0x0a)
+                return null;
         }
-        if (typeof record !== 'object' || record === null)
-            continue;
-        if (scanner.accept(record, lineOffset) === 'excluded') {
-            // Non-interactive Codex session: nothing in it will ever count.
+    }
+    return { head: { bytes: head.length, hash: hashBytes(head) }, bytesRead };
+}
+async function scanFile(candidate, stat, prior, budget, context) {
+    const handle = await fs.promises.open(candidate.filePath, 'r');
+    try {
+        let bytesRead = 0;
+        const verified = await verifyFile(handle, stat.size, prior);
+        let plan = resumePlan(prior);
+        let head = verified?.head;
+        if (verified) {
+            bytesRead += verified.bytesRead;
+        }
+        else {
+            // Offsets and ordinals of a rewritten file are meaningless: start over.
+            if (prior)
+                context.stats.sessionsRewritten++;
+            plan = { kind: 'fresh' };
+            const fresh = await verifyFile(handle, stat.size, undefined);
+            if (!fresh)
+                throw new Error('file changed while reading');
+            bytesRead += fresh.bytesRead;
+            head = fresh.head;
+        }
+        const session = plan.kind === 'fresh' ? undefined : plan.session;
+        if (session?.state?.excluded) {
+            return {
+                entries: [],
+                progressed: true,
+                session: {
+                    provider: candidate.provider,
+                    sessionId: candidate.sessionId,
+                    size: stat.size,
+                    mtimeMs: stat.mtimeMs,
+                    lastOrdinal: session.lastOrdinal,
+                    offset: stat.size,
+                    state: session.state,
+                    ...(head ? { head } : {}),
+                },
+                bytesRead,
+                stoppedBy: null,
+                notInteractive: true,
+                inScope: 0,
+                outOfScope: 0,
+            };
+        }
+        const scanner = candidate.provider === 'claude-code'
+            ? new ClaudeLineScanner(candidate, plan, context)
+            : new CodexLineScanner(candidate, plan, context);
+        const startOffset = plan.kind === 'fresh'
+            ? 0
+            : plan.kind === 'replay'
+                ? plan.pending.offset
+                : plan.session.offset;
+        const stream = await streamLines(handle, startOffset, stat.size, plan.kind === 'continue' && plan.session.skipping === true, budget, context, candidate, scanner);
+        bytesRead += stream.bytesRead;
+        if (stream.excluded) {
             return {
-                ...scanner.outcome(startOffset + buffer.length),
                 entries: [],
-                pending: undefined,
+                progressed: true,
+                session: {
+                    provider: candidate.provider,
+                    sessionId: candidate.sessionId,
+                    size: stat.size,
+                    mtimeMs: stat.mtimeMs,
+                    lastOrdinal: scanner.lastOrdinal(),
+                    offset: stat.size,
+                    state: scanner.resumeState() ?? { excluded: true },
+                    ...(head ? { head } : {}),
+                },
+                bytesRead,
+                stoppedBy: null,
                 notInteractive: true,
+                inScope: 0,
+                outOfScope: 0,
+            };
+        }
+        const update = await scanner.carriedUpdate(handle);
+        bytesRead += update.bytesRead;
+        if (update.rewritten) {
+            return {
+                entries: [],
+                progressed: false,
+                session: undefined,
+                bytesRead,
                 stoppedBy: null,
+                notInteractive: false,
+                inScope: 0,
+                outOfScope: 0,
             };
         }
+        const entries = scanner.returnedEntries();
+        if (update.entry)
+            entries.push(update.entry);
+        const state = scanner.resumeState();
+        const pending = scanner.pendingPrompt();
+        // A truncated read also keeps a backlog, so the next call re-verifies the file.
+        const backlog = stream.offset < stat.size && (stream.stoppedBy !== null || stream.truncated);
+        return {
+            entries,
+            progressed: stream.offset > startOffset || stream.skipping,
+            session: {
+                provider: candidate.provider,
+                sessionId: candidate.sessionId,
+                size: stat.size,
+                mtimeMs: stat.mtimeMs,
+                lastOrdinal: scanner.lastOrdinal(),
+                offset: stream.offset,
+                ...(state ? { state } : {}),
+                ...(pending ? { pending } : {}),
+                ...(backlog ? { backlog: true } : {}),
+                ...(stream.skipping ? { skipping: true } : {}),
+                ...(head ? { head } : {}),
+            },
+            bytesRead,
+            stoppedBy: backlog ? stream.stoppedBy : null,
+            notInteractive: false,
+            inScope: scanner.inScopeCount(),
+            outOfScope: scanner.outOfScopeCount(),
+        };
+    }
+    finally {
+        await handle.close();
+    }
+}
+/**
+ * Feeds complete JSONL lines from `[start, end)` to the scanner in fixed-size
+ * chunks. Lines split only at `\n`, which never occurs inside a multi-byte
+ * UTF-8 sequence, so decoding whole lines is exact whatever the chunk edges.
+ * A line longer than `maxRecordBytes` is dropped as it streams past.
+ */
+async function streamLines(handle, start, end, resumeSkipping, budget, context, candidate, scanner) {
+    const { maxRecordBytes, maxEntries } = context.bounds;
+    const chunk = Buffer.allocUnsafe(Math.max(1, Math.min(readChunkBytes, end - start || 1)));
+    let position = start;
+    let committed = start;
+    let lineStart = start;
+    let skipping = resumeSkipping;
+    let pieces = [];
+    let pieceBytes = 0;
+    let bytesRead = 0;
+    let lines = 0;
+    let stoppedBy = null;
+    let truncated = false;
+    // The first file of a call always finishes its first record, so no budget can stall it.
+    const mayStop = () => !(budget.forceProgress && committed === start && !skipping);
+    const result = (excluded = false) => ({
+        offset: skipping ? position : committed,
+        skipping,
+        bytesRead,
+        stoppedBy,
+        truncated,
+        excluded,
+    });
+    while (position < end) {
+        const stop = context.shouldStop();
+        if (stop) {
+            stoppedBy = stop;
+            break;
+        }
+        const allowance = budget.bytes - bytesRead;
+        if (allowance <= 0 && mayStop()) {
+            stoppedBy = budget.bound;
+            break;
+        }
+        const want = Math.min(chunk.length, end - position, allowance > 0 ? allowance : chunk.length);
+        const { bytesRead: got } = await handle.read(chunk, 0, want, position);
+        if (got === 0) {
+            // Truncated while reading; the next call's verification sees the rewrite.
+            truncated = true;
+            break;
+        }
+        bytesRead += got;
+        const view = chunk.subarray(0, got);
+        let index = 0;
+        while (index < got) {
+            const newline = view.indexOf(0x0a, index);
+            const segmentEnd = newline === -1 ? got : newline;
+            if (!skipping) {
+                const segmentBytes = segmentEnd - index;
+                if (pieceBytes + segmentBytes > maxRecordBytes) {
+                    skipping = true;
+                    pieces = [];
+                    pieceBytes = 0;
+                    context.stats.recordsOverSizeLimit++;
+                    context.exclude({
+                        provider: candidate.provider,
+                        sessionId: candidate.sessionId,
+                        reason: 'oversizedRecord',
+                        offset: lineStart,
+                    });
+                }
+                else if (newline === -1 && segmentBytes > 0) {
+                    // The chunk buffer is reused, so a partial line keeps a copy.
+                    pieces.push(Buffer.from(view.subarray(index, segmentEnd)));
+                    pieceBytes += segmentBytes;
+                }
+            }
+            if (newline === -1)
+                break;
+            if (skipping) {
+                skipping = false;
+            }
+            else {
+                let line;
+                if (pieces.length === 0) {
+                    line = view.toString('utf8', index, newline);
+                }
+                else {
+                    pieces.push(view.subarray(index, newline));
+                    line = Buffer.concat(pieces).toString('utf8');
+                }
+                const lineOffset = lineStart;
+                const lineBytes = position + newline - lineStart;
+                if (acceptLine(line, lineOffset, lineBytes, scanner, context, candidate) === 'excluded') {
+                    committed = position + newline + 1;
+                    return result(true);
+                }
+            }
+            pieces = [];
+            pieceBytes = 0;
+            index = newline + 1;
+            committed = position + index;
+            lineStart = committed;
+            lines++;
+            if (mayStop() && context.entriesSoFar() + scanner.entryCount() >= maxEntries) {
+                stoppedBy = 'maxEntries';
+                position += index;
+                return result();
+            }
+            if (lines % CHECK_EVERY_LINES === 0) {
+                await yieldToEventLoop();
+                const stop = context.shouldStop();
+                if (stop) {
+                    stoppedBy = stop;
+                    position += index;
+                    return result();
+                }
+            }
+        }
+        position += got;
     }
-    return { ...scanner.outcome(startOffset + completeLength), stoppedBy: null };
+    return result();
+}
+function acceptLine(line, lineOffset, lineBytes, scanner, context, candidate) {
+    if (!line.trim())
+        return 'ok';
+    let record;
+    try {
+        record = JSON.parse(line);
+    }
+    catch {
+        context.stats.recordsMalformed++;
+        context.exclude({
+            provider: candidate.provider,
+            sessionId: candidate.sessionId,
+            reason: 'malformedRecord',
+            offset: lineOffset,
+        });
+        return 'ok';
+    }
+    if (typeof record !== 'object' || record === null)
+        return 'ok';
+    return scanner.accept(record, lineOffset, lineBytes);
 }
 /** Shared per-file prompt bookkeeping for both providers. */
 class LineScanner {
@@ -417,45 +740,155 @@ class LineScanner {
     outOfScope = 0;
     /** Latest emitted prompt still collecting answer metadata. */
     open = null;
-    constructor(candidate, start, context) {
+    /** The carried prompt once it has been closed during this pass. */
+    closedCarried = null;
+    /** Version 2 replay: the prompt already returned with this digest is not returned again. */
+    suppress = null;
+    constructor(candidate, plan, context) {
         this.candidate = candidate;
         this.context = context;
-        this.nextOrdinal = start.replay ? start.replay.ordinal : (start.resume?.lastOrdinal ?? -1) + 1;
+        if (plan.kind === 'fresh') {
+            this.nextOrdinal = 0;
+        }
+        else if (plan.kind === 'replay') {
+            this.nextOrdinal = plan.pending.ordinal;
+            this.suppress = { ordinal: plan.pending.ordinal, emitted: plan.pending.emitted };
+        }
+        else {
+            this.nextOrdinal = plan.session.lastOrdinal + 1;
+            const carried = plan.carried;
+            if (carried?.answer &&
+                carried.lineBytes !== undefined &&
+                carried.lineBytes <= context.bounds.maxRecordBytes &&
+                carried.timestamp !== undefined) {
+                const answer = carried.answer;
+                const usage = answer.usage ? compactUsage(answer.usage) : undefined;
+                this.open = {
+                    entry: {
+                        provider: candidate.provider,
+                        sessionId: candidate.sessionId,
+                        ordinal: carried.ordinal,
+                        timestamp: carried.timestamp,
+                        text: '',
+                        cwd: '',
+                        ...(typeof answer.model === 'string' ? { model: answer.model } : {}),
+                        ...(usage ? { usage } : {}),
+                        metadataStatus: answer.status,
+                    },
+                    lineOffset: carried.offset,
+                    lineBytes: carried.lineBytes,
+                    ...(carried.state ? { stateBefore: { ...carried.state } } : {}),
+                    answerId: typeof answer.id === 'string' ? answer.id : null,
+                    carried: { emitted: carried.emitted },
+                };
+            }
+        }
     }
-    state() {
+    resumeState() {
         return undefined;
     }
-    outcome(offset) {
-        const state = this.state();
+    lastOrdinal() {
+        return this.nextOrdinal - 1;
+    }
+    entryCount() {
+        return this.entries.length;
+    }
+    inScopeCount() {
+        return this.inScope;
+    }
+    outOfScopeCount() {
+        return this.outOfScope;
+    }
+    returnedEntries() {
+        const suppress = this.suppress;
+        if (!suppress)
+            return [...this.entries];
+        // A replayed prompt is returned again only when its metadata changed.
+        return this.entries.filter((entry) => entry.ordinal !== suppress.ordinal || metadataDigest(entry) !== suppress.emitted);
+    }
+    pendingPrompt() {
         const open = this.open;
+        if (!open)
+            return undefined;
+        const entry = open.entry;
+        const status = entry.metadataStatus === 'provisional' ? 'provisional' : 'pending';
+        const answer = {
+            status,
+            ...(open.answerId !== null ? { id: open.answerId } : {}),
+            ...(entry.model ? { model: entry.model } : {}),
+            ...(entry.usage ? { usage: { ...entry.usage } } : {}),
+        };
         return {
-            entries: this.entries,
-            lastOrdinal: this.nextOrdinal - 1,
-            offset,
-            ...(state ? { state } : {}),
-            ...(open
-                ? {
-                    pending: {
-                        ordinal: open.entry.ordinal,
-                        offset: open.lineOffset,
-                        emitted: metadataDigest(open.entry),
-                        ...(open.stateBefore ? { state: open.stateBefore } : {}),
-                    },
-                }
-                : {}),
-            inScope: this.inScope,
-            outOfScope: this.outOfScope,
-            notInteractive: false,
+            ordinal: entry.ordinal,
+            offset: open.lineOffset,
+            lineBytes: open.lineBytes,
+            timestamp: entry.timestamp,
+            emitted: metadataDigest(entry),
+            ...(open.stateBefore ? { state: { ...open.stateBefore } } : {}),
+            answer,
+        };
+    }
+    /**
+     * The carried prompt returned again when its metadata changed in this pass.
+     * Its text is re-read from its own line, which must still match exactly.
+     */
+    async carriedUpdate(handle) {
+        const carried = this.closedCarried ?? (this.open?.carried ? this.open : null);
+        if (!carried?.carried)
+            return { bytesRead: 0, rewritten: false };
+        const entry = carried.entry;
+        if (metadataDigest(entry) === carried.carried.emitted)
+            return { bytesRead: 0, rewritten: false };
+        const raw = await readAt(handle, carried.lineOffset, carried.lineBytes + 1);
+        const rewritten = { bytesRead: raw.length, rewritten: true };
+        if (raw.length !== carried.lineBytes + 1 || raw[carried.lineBytes] !== 0x0a)
+            return rewritten;
+        let record;
+        try {
+            record = JSON.parse(raw.toString('utf8', 0, carried.lineBytes));
+        }
+        catch {
+            return rewritten;
+        }
+        if (typeof record !== 'object' || record === null)
+            return rewritten;
+        const row = record;
+        const text = this.promptTextOf(row);
+        if (text === null || row.timestamp !== entry.timestamp)
+            return rewritten;
+        const place = this.promptPlaceOf(row, carried.stateBefore);
+        const scoped = scopedCwd(place.cwd, this.context.roots, this.context.realpaths);
+        if (!scoped) {
+            // The directory no longer resolves inside the roots: fail closed.
+            this.context.stats.promptsOutOfScope++;
+            return { bytesRead: raw.length, rewritten: false };
+        }
+        if (carried === this.open)
+            carried.carried = { emitted: metadataDigest(entry) };
+        return {
+            entry: {
+                ...entry,
+                text,
+                cwd: scoped,
+                ...(place.gitBranch ? { gitBranch: place.gitBranch } : {}),
+                ...(entry.usage ? { usage: { ...entry.usage } } : {}),
+            },
+            bytesRead: raw.length,
+            rewritten: false,
         };
     }
     /** The open prompt's answer is complete, or will never arrive. */
     closeOpen() {
-        if (this.open)
-            this.open.entry.metadataStatus = 'final';
+        const open = this.open;
+        if (!open)
+            return;
+        open.entry.metadataStatus = 'final';
+        if (open.carried)
+            this.closedCarried = open;
         this.open = null;
     }
     /** Assign an ordinal to a human prompt and emit it when in scope. */
-    recordPrompt(text, timestamp, cwd, gitBranch, lineOffset, stateBefore) {
+    recordPrompt(text, timestamp, cwd, gitBranch, lineOffset, lineBytes, stateBefore) {
         const ordinal = this.nextOrdinal++;
         this.closeOpen();
         if (typeof timestamp !== 'string' || Number.isNaN(Date.parse(timestamp))) {
@@ -482,7 +915,13 @@ class LineScanner {
             metadataStatus: 'pending',
         };
         this.entries.push(entry);
-        this.open = { entry, lineOffset, ...(stateBefore ? { stateBefore } : {}) };
+        this.open = {
+            entry,
+            lineOffset,
+            lineBytes,
+            answerId: null,
+            ...(stateBefore ? { stateBefore } : {}),
+        };
     }
 }
 function transcriptMessage(candidate, record, role, content, extra, eventIndex) {
@@ -520,37 +959,48 @@ function compactUsage(usage) {
     }
     return Object.keys(result).length > 0 ? result : undefined;
 }
+function claudeUserText(candidate, record, message, blocks, eventIndex) {
+    const origin = record.origin;
+    const canonical = transcriptMessage(candidate, record, typeof message.role === 'string' ? message.role : 'user', blocks, {
+        entrypoint: stringField(record.entrypoint),
+        isMeta: record.isMeta === true ? true : undefined,
+        isSidechain: record.isSidechain === true ? true : undefined,
+        isCompactSummary: record.isCompactSummary === true ? true : undefined,
+        originKind: origin && typeof origin.kind === 'string' ? origin.kind : undefined,
+        promptSource: stringField(record.promptSource),
+    }, eventIndex);
+    return (0, humanPrompt_1.humanPromptText)(canonical, 'claude-code');
+}
 class ClaudeLineScanner extends LineScanner {
     lineIndex = 0;
-    answerId = null;
-    accept(record, lineOffset) {
+    promptTextOf(record) {
+        const message = record.message;
+        if (record.type !== 'user' || typeof message !== 'object' || message === null)
+            return null;
+        return claudeUserText(this.candidate, record, message, claudeBlocks(message.content), 0);
+    }
+    promptPlaceOf(record) {
+        return { cwd: record.cwd, gitBranch: stringField(record.gitBranch) };
+    }
+    accept(record, lineOffset, lineBytes) {
         const index = this.lineIndex++;
         const message = record.message;
         if (typeof message !== 'object' || message === null)
             return 'ok';
         if (record.type === 'user') {
-            const origin = record.origin;
             const blocks = claudeBlocks(message.content);
-            const canonical = transcriptMessage(this.candidate, record, typeof message.role === 'string' ? message.role : 'user', blocks, {
-                entrypoint: stringField(record.entrypoint),
-                isMeta: record.isMeta === true ? true : undefined,
-                isSidechain: record.isSidechain === true ? true : undefined,
-                isCompactSummary: record.isCompactSummary === true ? true : undefined,
-                originKind: origin && typeof origin.kind === 'string' ? origin.kind : undefined,
-                promptSource: stringField(record.promptSource),
-            }, index);
-            const text = (0, humanPrompt_1.humanPromptText)(canonical, 'claude-code');
+            const text = claudeUserText(this.candidate, record, message, blocks, index);
             if (text === null) {
                 // A tool result means the first answering call already finished.
-                if (this.answerId !== null &&
+                if (this.open !== null &&
+                    this.open.answerId !== null &&
                     record.isSidechain !== true &&
                     blocks.some((block) => block.type === 'tool_result')) {
                     this.closeOpen();
                 }
                 return 'ok';
             }
-            this.answerId = null;
-            this.recordPrompt(text, record.timestamp, record.cwd, stringField(record.gitBranch), lineOffset);
+            this.recordPrompt(text, record.timestamp, record.cwd, stringField(record.gitBranch), lineOffset, lineBytes);
             return 'ok';
         }
         const open = this.open;
@@ -559,10 +1009,10 @@ class ClaudeLineScanner extends LineScanner {
         // Synthetic records (API errors, interrupts) are not model calls.
         if (message.model === '<synthetic>')
             return 'ok';
-        const id = stringField(message.id) ?? `line:${index}`;
-        if (this.answerId === null)
-            this.answerId = id;
-        if (id !== this.answerId) {
+        const id = stringField(message.id) ?? `line:${lineOffset}`;
+        if (open.answerId === null)
+            open.answerId = id;
+        if (id !== open.answerId) {
             // A later tool-loop call: the first answering call is over and keeps its attribution.
             this.closeOpen();
             return 'ok';
@@ -618,22 +1068,36 @@ function claudeBlocks(content) {
 }
 class CodexLineScanner extends LineScanner {
     lineIndex = 0;
-    resumeState;
-    constructor(candidate, start, context) {
-        super(candidate, start, context);
+    state;
+    constructor(candidate, plan, context) {
+        super(candidate, plan, context);
         // A replay restarts at the open prompt's line, so it needs the state from just before it.
-        const initial = start.replay ? start.replay.state : start.resume?.state;
-        this.resumeState = { ...(initial ?? {}) };
+        const initial = plan.kind === 'replay'
+            ? plan.pending.state
+            : plan.kind === 'continue'
+                ? plan.session.state
+                : undefined;
+        this.state = { ...(initial ?? {}) };
     }
-    state() {
-        return { ...this.resumeState };
+    resumeState() {
+        return { ...this.state };
     }
-    accept(record, lineOffset) {
+    promptTextOf(record) {
+        const payload = record.payload;
+        if (!isCodexUserMessage(record, payload))
+            return null;
+        const canonical = transcriptMessage(this.candidate, record, 'user', codexBlocks(payload.content), {}, 0);
+        return (0, humanPrompt_1.humanPromptText)(canonical, 'codex');
+    }
+    promptPlaceOf(_record, stateBefore) {
+        return { cwd: stateBefore?.cwd, gitBranch: stateBefore?.gitBranch };
+    }
+    accept(record, lineOffset, lineBytes) {
         const index = this.lineIndex++;
         const payload = record.payload;
         if (typeof payload !== 'object' || payload === null)
             return 'ok';
-        const state = this.resumeState;
+        const state = this.state;
         if (record.type === 'session_meta') {
             // Forked sessions embed their parent's session_meta after their own; only the first counts.
             if (state.meta)
@@ -658,7 +1122,7 @@ class CodexLineScanner extends LineScanner {
                 state.model = model;
             return 'ok';
         }
-        if (record.type === 'response_item' && payload.type === 'message' && payload.role === 'user') {
+        if (isCodexUserMessage(record, payload)) {
             // Without session metadata the source (and thus interactivity) is unknown: fail closed.
             if (!state.meta) {
                 state.excluded = true;
@@ -668,7 +1132,7 @@ class CodexLineScanner extends LineScanner {
             const text = (0, humanPrompt_1.humanPromptText)(canonical, 'codex');
             if (text === null)
                 return 'ok';
-            this.recordPrompt(text, record.timestamp, state.cwd, state.gitBranch, lineOffset, {
+            this.recordPrompt(text, record.timestamp, state.cwd, state.gitBranch, lineOffset, lineBytes, {
                 ...state,
             });
             return 'ok';
@@ -697,6 +1161,13 @@ class CodexLineScanner extends LineScanner {
         return 'ok';
     }
 }
+function isCodexUserMessage(record, payload) {
+    return (record.type === 'response_item' &&
+        typeof payload === 'object' &&
+        payload !== null &&
+        payload.type === 'message' &&
+        payload.role === 'user');
+}
 function codexBlocks(content) {
     if (!Array.isArray(content))
         return [];