sidekick-shared 0.28.3 → 0.29.1

package/dist/esm/watchers/jsonlTail.js

@@ -11,29 +11,112 @@ const DEFAULT_CATCHUP_INTERVAL_MS = 30_000;
 export const DEFAULT_JSONL_TAIL_READ_BYTES = 1024 * 1024;
 /** Default longest line parsed; longer lines are skipped and reported. */
 export const DEFAULT_JSONL_TAIL_MAX_LINE_BYTES = 16 * 1024 * 1024;
-/** Skip old complete events while retaining an in-progress final line. */
-function lastCompleteLineOffset(filePath) {
-    const fd = fs.openSync(filePath, 'r');
-    try {
-        let end = fs.fstatSync(fd).size;
-        const buffer = Buffer.alloc(64 * 1024);
-        while (end > 0) {
-            const start = Math.max(0, end - buffer.length);
-            const count = fs.readSync(fd, buffer, 0, end - start, start);
-            const newline = buffer.subarray(0, count).lastIndexOf(0x0a);
-            if (newline >= 0)
-                return start + newline + 1;
-            end = start;
-        }
-        return 0;
+/** The `version` written into, and accepted from, {@link JsonlTailCheckpoint}. */
+export const JSONL_TAIL_CHECKPOINT_VERSION = 1;
+const BACKWARD_SCAN_BYTES = 64 * 1024;
+/**
+ * Skip old complete events while retaining an in-progress final line: the
+ * offset just after the last newline in the first `size` bytes, or 0.
+ */
+function lastCompleteLineOffset(fd, size) {
+    let end = size;
+    const buffer = Buffer.alloc(BACKWARD_SCAN_BYTES);
+    while (end > 0) {
+        const start = Math.max(0, end - buffer.length);
+        const count = fs.readSync(fd, buffer, 0, end - start, start);
+        const newline = buffer.subarray(0, count).lastIndexOf(0x0a);
+        if (newline >= 0)
+            return start + newline + 1;
+        end = start;
     }
-    finally {
-        fs.closeSync(fd);
+    return 0;
+}
+async function lastCompleteLineOffsetAsync(handle, size) {
+    let end = size;
+    const buffer = Buffer.alloc(BACKWARD_SCAN_BYTES);
+    while (end > 0) {
+        const start = Math.max(0, end - buffer.length);
+        const { bytesRead } = await handle.read(buffer, 0, end - start, start);
+        const newline = buffer.subarray(0, bytesRead).lastIndexOf(0x0a);
+        if (newline >= 0)
+            return start + newline + 1;
+        end = start;
+    }
+    return 0;
+}
+/**
+ * Reported to `onError` (and nothing is read) when `noFollow` is set and the
+ * tailed path is a symbolic link.
+ */
+export class JsonlTailSymlinkError extends Error {
+    path;
+    code = 'JSONL_TAIL_SYMLINK';
+    constructor(path) {
+        super(`Refusing to follow the symbolic link at ${path} (noFollow is set).`);
+        this.path = path;
+        this.name = 'JsonlTailSymlinkError';
     }
 }
 export function createJsonlTail(options) {
     return new JsonlTailReader(options);
 }
+/**
+ * Records a {@link JsonlTailCheckpoint} for byte `offset` of `filePath`
+ * without a tail, for readers that keep their own offset (a snapshot that
+ * stores a reader position, for example). The checkpoint carries the file's
+ * current device and inode and a digest of its first bytes, so
+ * {@link verifyJsonlCheckpoint} (or {@link JsonlTail.resumeFrom}) can later
+ * tell whether `offset` still refers to the same, append-only content.
+ *
+ * Returns `null` when the file is missing or unreadable, or when `offset` is
+ * not a resume-safe position in it: negative, past the end, or not 0 and not
+ * just after a newline. Follows symbolic links.
+ */
+export function captureJsonlCheckpoint(filePath, offset) {
+    if (!Number.isSafeInteger(offset) || offset < 0)
+        return null;
+    let fd = null;
+    try {
+        fd = fs.openSync(filePath, 'r');
+        const stat = fs.fstatSync(fd);
+        if (offset > stat.size)
+            return null;
+        if (offset > 0 && readAt(fd, offset - 1, 1)[0] !== 0x0a)
+            return null;
+        return checkpointOf(offset, stat, fileHeadOf(readPrefix(fd, stat)));
+    }
+    catch {
+        return null;
+    }
+    finally {
+        closeQuietly(fd);
+    }
+}
+/**
+ * Whether `checkpoint` (any value, typically parsed from JSON) is a valid
+ * {@link JsonlTailCheckpoint} that still describes `filePath`: same device
+ * and inode, the recorded first bytes unchanged, and its offset within the
+ * file and just after a newline (or 0). This is the check
+ * {@link JsonlTail.resumeFrom} applies. `false` on any error, including a
+ * missing file. Follows symbolic links.
+ */
+export function verifyJsonlCheckpoint(filePath, checkpoint) {
+    const parsed = parseCheckpoint(checkpoint);
+    if (parsed === null)
+        return false;
+    let fd = null;
+    try {
+        fd = fs.openSync(filePath, 'r');
+        const stat = fs.fstatSync(fd);
+        return checkpointMatches(parsed, fd, stat, readPrefix(fd, stat));
+    }
+    catch {
+        return false;
+    }
+    finally {
+        closeQuietly(fd);
+    }
+}
 class JsonlTailReader {
     options;
     active = false;
@@ -51,20 +134,34 @@ class JsonlTailReader {
     lastMtimeMs = -1;
     maxReadBytes;
     maxLineBytes;
+    rewritePolicy;
     fsWatcher = null;
     debounceTimer = null;
     catchupTimer = null;
     eventsInCurrentBatch = 0;
     parser;
+    /** Collects parsed events while a `read()` is running. */
+    collector = null;
+    /** Serializes `read()` calls. */
+    readChain = Promise.resolve();
+    /** `read()` calls queued or running; `readNow()` waits for them. */
+    queuedReads = 0;
+    readNowDeferred = false;
+    /** Bumped by repositioning calls so an in-flight `read()` stops early. */
+    generation = 0;
+    /** `resumeFrom()` chose the position; `start()` must not move it to the end. */
+    positioned = false;
     constructor(options) {
         this.options = options;
         this.readOffset = options.startOffset ?? 0;
         this.committedOffset = this.readOffset;
         this.maxReadBytes = positiveInteger(options.maxReadBytes, DEFAULT_JSONL_TAIL_READ_BYTES);
         this.maxLineBytes = positiveInteger(options.maxLineBytes, DEFAULT_JSONL_TAIL_MAX_LINE_BYTES);
+        this.rewritePolicy = options.onRewrite === 'end' ? 'end' : 'restart';
         this.parser = new JsonlParser({
             onEvent: (event) => {
                 this.eventsInCurrentBatch += 1;
+                this.collector?.push({ event, line: this.currentLine });
                 this.options.onEvent(event, this.currentLine);
             },
             onError: (error, line) => this.options.onError?.(error, line),
@@ -77,13 +174,18 @@ class JsonlTailReader {
         if (this.active)
             return;
         this.active = true;
-        if (this.options.startAtEnd && this.options.startOffset === undefined) {
+        if (this.options.startAtEnd && this.options.startOffset === undefined && !this.positioned) {
+            let fd = null;
             try {
-                this.readOffset = lastCompleteLineOffset(this.options.path);
+                fd = this.openSync();
+                this.readOffset = lastCompleteLineOffset(fd, fs.fstatSync(fd).size);
             }
             catch {
                 this.readOffset = 0;
             }
+            finally {
+                closeQuietly(fd);
+            }
             this.committedOffset = this.readOffset;
         }
         this.readNow();
@@ -115,11 +217,20 @@ class JsonlTailReader {
         this.stop();
     }
     readNow() {
+        if (this.queuedReads > 0) {
+            this.readNowDeferred = true;
+            return;
+        }
         let fd = null;
         try {
-            fd = fs.openSync(this.options.path, 'r');
+            fd = this.openSync();
             const stat = fs.fstatSync(fd);
-            this.checkContinuity(fd, stat);
+            const probe = this.probeNeeded(stat) ? this.probeSync(fd, stat) : null;
+            const reason = this.resetReason(stat, probe);
+            if (reason !== null) {
+                this.reset(reason, this.rewritePolicy === 'end' ? lastCompleteLineOffset(fd, stat.size) : 0);
+            }
+            this.adoptFile(stat, probe);
             if (stat.size <= this.readOffset)
                 return;
             this.eventsInCurrentBatch = 0;
@@ -143,61 +254,166 @@ class JsonlTailReader {
             });
         }
         catch (error) {
-            this.options.onError?.(error instanceof Error ? error : new Error(String(error)));
+            this.options.onError?.(toError(error));
         }
         finally {
-            if (fd !== null) {
-                try {
-                    fs.closeSync(fd);
-                }
-                catch {
-                    /* ignore close errors */
+            closeQuietly(fd);
+        }
+    }
+    read() {
+        this.queuedReads += 1;
+        const run = this.readChain.then(() => this.readAsync());
+        this.readChain = run.catch(() => undefined);
+        return run;
+    }
+    async readAsync() {
+        const generation = this.generation;
+        const events = [];
+        let reset;
+        let bytesRead = 0;
+        let eventsRead = 0;
+        let handle = null;
+        try {
+            handle = await this.openAsync();
+            const stat = await handle.stat();
+            const probe = this.probeNeeded(stat) ? await this.probeAsync(handle, stat) : null;
+            if (generation === this.generation) {
+                const reason = this.resetReason(stat, probe);
+                const offset = reason !== null && this.rewritePolicy === 'end'
+                    ? await lastCompleteLineOffsetAsync(handle, stat.size)
+                    : 0;
+                if (generation === this.generation) {
+                    if (reason !== null)
+                        reset = this.reset(reason, offset);
+                    this.adoptFile(stat, probe);
+                    if (stat.size > this.readOffset) {
+                        this.eventsInCurrentBatch = 0;
+                        const chunk = Buffer.alloc(Math.min(this.maxReadBytes, stat.size - this.readOffset));
+                        while (this.readOffset < stat.size) {
+                            const want = Math.min(chunk.length, stat.size - this.readOffset);
+                            const position = this.readOffset;
+                            const { bytesRead: count } = await handle.read(chunk, 0, want, position);
+                            if (count <= 0 || generation !== this.generation)
+                                break;
+                            this.collector = events;
+                            try {
+                                this.consume(chunk.subarray(0, count), position);
+                            }
+                            finally {
+                                this.collector = null;
+                            }
+                            this.readOffset += count;
+                            bytesRead += count;
+                        }
+                        eventsRead = this.eventsInCurrentBatch;
+                        if (bytesRead > 0) {
+                            this.options.onBatchComplete?.({
+                                bytesRead,
+                                eventsRead,
+                                offset: this.committedOffset,
+                            });
+                        }
+                    }
                 }
             }
         }
+        catch (error) {
+            this.options.onError?.(toError(error));
+        }
+        finally {
+            if (handle)
+                await handle.close().catch(() => undefined);
+            this.queuedReads -= 1;
+            if (this.queuedReads === 0 && this.readNowDeferred) {
+                this.readNowDeferred = false;
+                this.readNow();
+            }
+        }
+        return {
+            bytesRead,
+            eventsRead,
+            offset: this.committedOffset,
+            events,
+            ...(reset ? { reset } : {}),
+        };
+    }
+    /** Whether the continuity check needs bytes from the file. */
+    probeNeeded(stat) {
+        return (this.identity === null ||
+            stat.dev !== this.identity.dev ||
+            stat.ino !== this.identity.ino ||
+            stat.size < this.readOffset ||
+            stat.mtimeMs !== this.lastMtimeMs ||
+            this.head === null ||
+            this.head.bytes < Math.min(HEAD_BYTES, stat.size));
+    }
+    probeSync(fd, stat) {
+        return {
+            prefix: readPrefix(fd, stat),
+            before: this.committedOffset > 0 && this.committedOffset <= stat.size
+                ? readAt(fd, this.committedOffset - 1, 1)
+                : null,
+        };
+    }
+    async probeAsync(handle, stat) {
+        const committedOffset = this.committedOffset;
+        return {
+            prefix: await readAtAsync(handle, 0, Math.min(HEAD_BYTES, stat.size)),
+            before: committedOffset > 0 && committedOffset <= stat.size
+                ? await readAtAsync(handle, committedOffset - 1, 1)
+                : null,
+        };
     }
     /**
-     * Starts over when the path now names another file, the file shrank, its
-     * first bytes changed, or the resume point no longer follows a newline.
+     * Why the tail must reposition: the path now names another file, the file
+     * shrank, its first bytes changed, or the resume point no longer follows a
+     * newline. `null` when it can continue.
      */
-    checkContinuity(fd, stat) {
-        let reason = null;
+    resetReason(stat, probe) {
         if (this.identity && (stat.dev !== this.identity.dev || stat.ino !== this.identity.ino)) {
-            reason = 'replaced';
+            return 'replaced';
         }
-        else if (stat.size < this.readOffset) {
-            reason = 'truncated';
+        if (stat.size < this.readOffset)
+            return 'truncated';
+        if (this.identity && stat.mtimeMs !== this.lastMtimeMs && probe) {
+            if (this.head && !headMatches(this.head, probe.prefix))
+                return 'rewritten';
+            if (probe.before && probe.before[0] !== 0x0a)
+                return 'rewritten';
         }
-        if (reason === null && this.identity && stat.mtimeMs !== this.lastMtimeMs) {
-            const prefix = readAt(fd, 0, Math.min(HEAD_BYTES, stat.size));
-            const before = this.committedOffset > 0 ? readAt(fd, this.committedOffset - 1, 1) : null;
-            if ((this.head && !headMatches(this.head, prefix)) || (before && before[0] !== 0x0a)) {
-                reason = 'rewritten';
-            }
-        }
-        if (reason !== null)
-            this.reset(reason);
+        return null;
+    }
+    /** Records the identity and prefix of the file the offsets now refer to. */
+    adoptFile(stat, probe) {
         this.identity = { dev: stat.dev, ino: stat.ino };
         this.lastMtimeMs = stat.mtimeMs;
-        if (!this.head || this.head.bytes < Math.min(HEAD_BYTES, stat.size)) {
-            this.head = fileHeadOf(readAt(fd, 0, Math.min(HEAD_BYTES, stat.size)));
+        if (probe && (!this.head || this.head.bytes < Math.min(HEAD_BYTES, stat.size))) {
+            this.head = fileHeadOf(probe.prefix);
         }
     }
-    reset(reason) {
+    reset(reason, offset) {
         const previousOffset = this.committedOffset;
-        this.readOffset = 0;
-        this.committedOffset = 0;
-        this.pendingBytes = Buffer.alloc(0);
-        this.skippingLine = false;
+        this.position(offset);
         this.head = null;
-        this.parser.reset();
         if (reason === 'replaced' && this.active) {
             // fs.watch follows the old inode; watch the file now at the path.
             this.fsWatcher?.close();
             this.fsWatcher = null;
             this.watchFile();
         }
-        this.options.onReset?.({ reason, previousOffset });
+        const reset = this.rewritePolicy === 'end'
+            ? { reason, previousOffset, offset }
+            : { reason, previousOffset };
+        this.options.onReset?.(reset);
+        return reset;
+    }
+    /** Moves both offsets to `offset` and drops any partial-line state. */
+    position(offset) {
+        this.readOffset = offset;
+        this.committedOffset = offset;
+        this.pendingBytes = Buffer.alloc(0);
+        this.skippingLine = false;
+        this.parser.reset();
     }
     /** Parses the complete lines in `bytes`, which start at file offset `position`. */
     consume(bytes, position) {
@@ -249,13 +465,93 @@ class JsonlTailReader {
         return this.committedOffset;
     }
     seekTo(offset) {
-        this.readOffset = Math.max(0, offset);
-        this.committedOffset = this.readOffset;
-        this.pendingBytes = Buffer.alloc(0);
-        this.skippingLine = false;
-        this.parser.reset();
+        this.generation += 1;
+        this.position(Math.max(0, offset));
+    }
+    checkpoint() {
+        return checkpointOf(this.committedOffset, this.identity, this.head);
+    }
+    resumeFrom(checkpoint) {
+        this.generation += 1;
+        this.positioned = true;
+        const parsed = parseCheckpoint(checkpoint);
+        let fd = null;
+        try {
+            fd = this.openSync();
+            const stat = fs.fstatSync(fd);
+            const prefix = readPrefix(fd, stat);
+            const matches = parsed !== null && checkpointMatches(parsed, fd, stat, prefix);
+            const offset = matches
+                ? parsed.offset
+                : this.rewritePolicy === 'end'
+                    ? lastCompleteLineOffset(fd, stat.size)
+                    : 0;
+            this.position(offset);
+            this.identity = { dev: stat.dev, ino: stat.ino };
+            this.lastMtimeMs = stat.mtimeMs;
+            this.head = fileHeadOf(prefix);
+            return matches;
+        }
+        catch (error) {
+            this.position(0);
+            this.identity = null;
+            this.head = null;
+            this.lastMtimeMs = -1;
+            this.options.onError?.(toError(error));
+            return false;
+        }
+        finally {
+            closeQuietly(fd);
+        }
+    }
+    openFlags() {
+        const noFollow = this.options.noFollow ? fs.constants.O_NOFOLLOW : undefined;
+        return typeof noFollow === 'number' ? fs.constants.O_RDONLY | noFollow : 'r';
+    }
+    openSync() {
+        try {
+            return fs.openSync(this.options.path, this.openFlags());
+        }
+        catch (error) {
+            throw this.refusedSymlink(error) ?? error;
+        }
+    }
+    async openAsync() {
+        try {
+            return await fs.promises.open(this.options.path, this.openFlags());
+        }
+        catch (error) {
+            throw this.refusedSymlink(error) ?? error;
+        }
+    }
+    /** The typed error for an open that `O_NOFOLLOW` refused because the path is a symlink. */
+    refusedSymlink(error) {
+        if (!this.options.noFollow)
+            return null;
+        const code = error?.code;
+        // ELOOP on Linux and macOS; EMLINK on FreeBSD.
+        if (code !== 'ELOOP' && code !== 'EMLINK')
+            return null;
+        try {
+            return fs.lstatSync(this.options.path).isSymbolicLink()
+                ? new JsonlTailSymlinkError(this.options.path)
+                : null;
+        }
+        catch {
+            return null;
+        }
     }
     watchFile() {
+        if (this.options.noFollow) {
+            try {
+                // fs.watch follows links; a refused symlink is not watched either.
+                if (fs.lstatSync(this.options.path).isSymbolicLink())
+                    return;
+            }
+            catch {
+                // Missing: fs.watch below fails the same way and polling covers it.
+            }
+        }
         try {
             this.fsWatcher = fs.watch(this.options.path, { persistent: false }, () => {
                 this.debouncedRead();
@@ -277,13 +573,101 @@ class JsonlTailReader {
         this.debounceTimer = setTimeout(() => this.readNow(), debounceMs);
     }
 }
+/** The checkpoint when `value` has the documented shape, else `null`. */
+function parseCheckpoint(value) {
+    if (!value || typeof value !== 'object')
+        return null;
+    const candidate = value;
+    if (candidate.version !== JSONL_TAIL_CHECKPOINT_VERSION)
+        return null;
+    const { offset, identity, prefixDigest } = candidate;
+    if (typeof offset !== 'number' || !Number.isSafeInteger(offset) || offset < 0)
+        return null;
+    let parsedIdentity = null;
+    if (identity !== null) {
+        if (!identity || typeof identity !== 'object')
+            return null;
+        const { dev, ino } = identity;
+        if (!isFiniteNumber(dev) || !isFiniteNumber(ino))
+            return null;
+        parsedIdentity = { dev, ino };
+    }
+    let parsedDigest = null;
+    if (prefixDigest !== null) {
+        if (!prefixDigest || typeof prefixDigest !== 'object')
+            return null;
+        const { bytes, hash } = prefixDigest;
+        if (typeof bytes !== 'number' || !Number.isInteger(bytes) || bytes < 0 || bytes > HEAD_BYTES) {
+            return null;
+        }
+        if (typeof hash !== 'string' || hash.length === 0)
+            return null;
+        parsedDigest = { bytes, hash };
+    }
+    return {
+        version: JSONL_TAIL_CHECKPOINT_VERSION,
+        offset,
+        identity: parsedIdentity,
+        prefixDigest: parsedDigest,
+    };
+}
+/** A fresh checkpoint for `offset` in the file with `identity` and recorded `head`. */
+function checkpointOf(offset, identity, head) {
+    return {
+        version: JSONL_TAIL_CHECKPOINT_VERSION,
+        offset,
+        identity: identity ? { dev: identity.dev, ino: identity.ino } : null,
+        prefixDigest: head ? { bytes: head.bytes, hash: head.hash } : null,
+    };
+}
+/** Whether `checkpoint` still describes the open file. */
+function checkpointMatches(checkpoint, fd, stat, prefix) {
+    const { offset, identity, prefixDigest } = checkpoint;
+    // Taken before the first read: nothing was consumed, so nothing can be stale.
+    if (identity === null && prefixDigest === null)
+        return offset === 0;
+    if (identity === null || prefixDigest === null)
+        return false;
+    if (identity.dev !== stat.dev || identity.ino !== stat.ino)
+        return false;
+    if (!headMatches(prefixDigest, prefix))
+        return false;
+    if (offset > stat.size)
+        return false;
+    return offset === 0 || readAt(fd, offset - 1, 1)[0] === 0x0a;
+}
+function isFiniteNumber(value) {
+    return typeof value === 'number' && Number.isFinite(value);
+}
 function positiveInteger(value, fallback) {
     return typeof value === 'number' && Number.isFinite(value) && value >= 1
         ? Math.floor(value)
         : fallback;
 }
+function toError(error) {
+    return error instanceof Error ? error : new Error(String(error));
+}
+function closeQuietly(fd) {
+    if (fd === null)
+        return;
+    try {
+        fs.closeSync(fd);
+    }
+    catch {
+        /* ignore close errors */
+    }
+}
 function readAt(fd, position, length) {
     const buffer = Buffer.alloc(length);
     const count = length > 0 ? fs.readSync(fd, buffer, 0, length, position) : 0;
     return buffer.subarray(0, count);
 }
+/** The file's first `min(HEAD_BYTES, size)` bytes: what a checkpoint digests. */
+function readPrefix(fd, stat) {
+    return readAt(fd, 0, Math.min(HEAD_BYTES, stat.size));
+}
+async function readAtAsync(handle, position, length) {
+    const buffer = Buffer.alloc(length);
+    const count = length > 0 ? (await handle.read(buffer, 0, length, position)).bytesRead : 0;
+    return buffer.subarray(0, count);
+}