sidekick-shared 0.27.6 → 0.28.0

package/dist/esm/watchers/jsonlTail.js

@@ -0,0 +1,289 @@
+/**
+ * Low-level JSONL tail reader for consumers that need raw parsed events plus
+ * their own aggregation lifecycle.
+ */
+import * as fs from 'fs';
+import { fileHeadOf, HEAD_BYTES, headMatches } from '../fileHead';
+import { JsonlParser } from '../parsers/jsonl';
+const DEFAULT_DEBOUNCE_MS = 100;
+const DEFAULT_CATCHUP_INTERVAL_MS = 30_000;
+/** Default bytes read from the file per chunk. */
+export const DEFAULT_JSONL_TAIL_READ_BYTES = 1024 * 1024;
+/** Default longest line parsed; longer lines are skipped and reported. */
+export const DEFAULT_JSONL_TAIL_MAX_LINE_BYTES = 16 * 1024 * 1024;
+/** Skip old complete events while retaining an in-progress final line. */
+function lastCompleteLineOffset(filePath) {
+    const fd = fs.openSync(filePath, 'r');
+    try {
+        let end = fs.fstatSync(fd).size;
+        const buffer = Buffer.alloc(64 * 1024);
+        while (end > 0) {
+            const start = Math.max(0, end - buffer.length);
+            const count = fs.readSync(fd, buffer, 0, end - start, start);
+            const newline = buffer.subarray(0, count).lastIndexOf(0x0a);
+            if (newline >= 0)
+                return start + newline + 1;
+            end = start;
+        }
+        return 0;
+    }
+    finally {
+        fs.closeSync(fd);
+    }
+}
+export function createJsonlTail(options) {
+    return new JsonlTailReader(options);
+}
+class JsonlTailReader {
+    options;
+    active = false;
+    /** Physical byte position already read from the file. */
+    readOffset;
+    /** Resume-safe byte position ending immediately after a newline. */
+    committedOffset;
+    pendingBytes = Buffer.alloc(0);
+    /** Inside a line over `maxLineBytes`: drop bytes until its newline. */
+    skippingLine = false;
+    currentLine = { offset: 0, length: 0 };
+    /** Identity and prefix of the file the offsets refer to, once read. */
+    identity = null;
+    head = null;
+    lastMtimeMs = -1;
+    maxReadBytes;
+    maxLineBytes;
+    fsWatcher = null;
+    debounceTimer = null;
+    catchupTimer = null;
+    eventsInCurrentBatch = 0;
+    parser;
+    constructor(options) {
+        this.options = options;
+        this.readOffset = options.startOffset ?? 0;
+        this.committedOffset = this.readOffset;
+        this.maxReadBytes = positiveInteger(options.maxReadBytes, DEFAULT_JSONL_TAIL_READ_BYTES);
+        this.maxLineBytes = positiveInteger(options.maxLineBytes, DEFAULT_JSONL_TAIL_MAX_LINE_BYTES);
+        this.parser = new JsonlParser({
+            onEvent: (event) => {
+                this.eventsInCurrentBatch += 1;
+                this.options.onEvent(event, this.currentLine);
+            },
+            onError: (error, line) => this.options.onError?.(error, line),
+        }, { schema: options.schema });
+    }
+    get isActive() {
+        return this.active;
+    }
+    start() {
+        if (this.active)
+            return;
+        this.active = true;
+        if (this.options.startAtEnd && this.options.startOffset === undefined) {
+            try {
+                this.readOffset = lastCompleteLineOffset(this.options.path);
+            }
+            catch {
+                this.readOffset = 0;
+            }
+            this.committedOffset = this.readOffset;
+        }
+        this.readNow();
+        this.watchFile();
+        const catchupIntervalMs = this.options.catchupIntervalMs ?? DEFAULT_CATCHUP_INTERVAL_MS;
+        if (catchupIntervalMs > 0) {
+            this.catchupTimer = setInterval(() => this.readNow(), catchupIntervalMs);
+        }
+    }
+    stop() {
+        if (!this.active)
+            return;
+        this.active = false;
+        if (this.debounceTimer) {
+            clearTimeout(this.debounceTimer);
+            this.debounceTimer = null;
+        }
+        if (this.fsWatcher) {
+            this.fsWatcher.close();
+            this.fsWatcher = null;
+        }
+        if (this.catchupTimer) {
+            clearInterval(this.catchupTimer);
+            this.catchupTimer = null;
+        }
+        this.parser.flush();
+    }
+    dispose() {
+        this.stop();
+    }
+    readNow() {
+        let fd = null;
+        try {
+            fd = fs.openSync(this.options.path, 'r');
+            const stat = fs.fstatSync(fd);
+            this.checkContinuity(fd, stat);
+            if (stat.size <= this.readOffset)
+                return;
+            this.eventsInCurrentBatch = 0;
+            let bytesRead = 0;
+            const chunk = Buffer.alloc(Math.min(this.maxReadBytes, stat.size - this.readOffset));
+            while (this.readOffset < stat.size) {
+                const want = Math.min(chunk.length, stat.size - this.readOffset);
+                const count = fs.readSync(fd, chunk, 0, want, this.readOffset);
+                if (count <= 0)
+                    break;
+                this.consume(chunk.subarray(0, count), this.readOffset);
+                this.readOffset += count;
+                bytesRead += count;
+            }
+            if (bytesRead <= 0)
+                return;
+            this.options.onBatchComplete?.({
+                bytesRead,
+                eventsRead: this.eventsInCurrentBatch,
+                offset: this.committedOffset,
+            });
+        }
+        catch (error) {
+            this.options.onError?.(error instanceof Error ? error : new Error(String(error)));
+        }
+        finally {
+            if (fd !== null) {
+                try {
+                    fs.closeSync(fd);
+                }
+                catch {
+                    /* ignore close errors */
+                }
+            }
+        }
+    }
+    /**
+     * Starts over when the path now names another file, the file shrank, its
+     * first bytes changed, or the resume point no longer follows a newline.
+     */
+    checkContinuity(fd, stat) {
+        let reason = null;
+        if (this.identity && (stat.dev !== this.identity.dev || stat.ino !== this.identity.ino)) {
+            reason = 'replaced';
+        }
+        else if (stat.size < this.readOffset) {
+            reason = 'truncated';
+        }
+        if (reason === null && this.identity && stat.mtimeMs !== this.lastMtimeMs) {
+            const prefix = readAt(fd, 0, Math.min(HEAD_BYTES, stat.size));
+            const before = this.committedOffset > 0 ? readAt(fd, this.committedOffset - 1, 1) : null;
+            if ((this.head && !headMatches(this.head, prefix)) || (before && before[0] !== 0x0a)) {
+                reason = 'rewritten';
+            }
+        }
+        if (reason !== null)
+            this.reset(reason);
+        this.identity = { dev: stat.dev, ino: stat.ino };
+        this.lastMtimeMs = stat.mtimeMs;
+        if (!this.head || this.head.bytes < Math.min(HEAD_BYTES, stat.size)) {
+            this.head = fileHeadOf(readAt(fd, 0, Math.min(HEAD_BYTES, stat.size)));
+        }
+    }
+    reset(reason) {
+        const previousOffset = this.committedOffset;
+        this.readOffset = 0;
+        this.committedOffset = 0;
+        this.pendingBytes = Buffer.alloc(0);
+        this.skippingLine = false;
+        this.head = null;
+        this.parser.reset();
+        if (reason === 'replaced' && this.active) {
+            // fs.watch follows the old inode; watch the file now at the path.
+            this.fsWatcher?.close();
+            this.fsWatcher = null;
+            this.watchFile();
+        }
+        this.options.onReset?.({ reason, previousOffset });
+    }
+    /** Parses the complete lines in `bytes`, which start at file offset `position`. */
+    consume(bytes, position) {
+        let start = 0;
+        if (this.skippingLine) {
+            const newline = bytes.indexOf(0x0a);
+            if (newline < 0)
+                return;
+            this.skippingLine = false;
+            start = newline + 1;
+            this.committedOffset = position + start;
+        }
+        let lineStart = this.committedOffset;
+        let cursor = start;
+        for (;;) {
+            const newline = bytes.indexOf(0x0a, cursor);
+            if (newline < 0)
+                break;
+            const piece = bytes.subarray(cursor, newline);
+            const line = this.pendingBytes.length > 0 ? Buffer.concat([this.pendingBytes, piece]) : piece;
+            this.pendingBytes = Buffer.alloc(0);
+            const lineEnd = position + newline + 1;
+            if (line.length > this.maxLineBytes) {
+                this.reportOversized(lineStart, line.length);
+            }
+            else {
+                this.currentLine = { offset: lineStart, length: line.length };
+                this.parser.processChunk(line.toString('utf8') + '\n');
+            }
+            this.committedOffset = lineEnd;
+            lineStart = lineEnd;
+            cursor = newline + 1;
+        }
+        const rest = bytes.subarray(cursor);
+        if (rest.length === 0)
+            return;
+        if (this.pendingBytes.length + rest.length > this.maxLineBytes) {
+            this.reportOversized(lineStart, this.pendingBytes.length + rest.length);
+            this.pendingBytes = Buffer.alloc(0);
+            this.skippingLine = true;
+            return;
+        }
+        this.pendingBytes = Buffer.concat([this.pendingBytes, rest]);
+    }
+    reportOversized(offset, atLeast) {
+        this.options.onError?.(new Error(`JSONL line at byte ${offset} is over ${this.maxLineBytes} bytes (${atLeast} so far); skipped.`));
+    }
+    getOffset() {
+        return this.committedOffset;
+    }
+    seekTo(offset) {
+        this.readOffset = Math.max(0, offset);
+        this.committedOffset = this.readOffset;
+        this.pendingBytes = Buffer.alloc(0);
+        this.skippingLine = false;
+        this.parser.reset();
+    }
+    watchFile() {
+        try {
+            this.fsWatcher = fs.watch(this.options.path, { persistent: false }, () => {
+                this.debouncedRead();
+            });
+            this.fsWatcher.on('error', (error) => {
+                this.fsWatcher?.close();
+                this.fsWatcher = null;
+                this.options.onError?.(error);
+            });
+        }
+        catch {
+            // Polling still covers filesystems where fs.watch is unavailable.
+        }
+    }
+    debouncedRead() {
+        if (this.debounceTimer)
+            clearTimeout(this.debounceTimer);
+        const debounceMs = this.options.debounceMs ?? DEFAULT_DEBOUNCE_MS;
+        this.debounceTimer = setTimeout(() => this.readNow(), debounceMs);
+    }
+}
+function positiveInteger(value, fallback) {
+    return typeof value === 'number' && Number.isFinite(value) && value >= 1
+        ? Math.floor(value)
+        : fallback;
+}
+function readAt(fd, position, length) {
+    const buffer = Buffer.alloc(length);
+    const count = length > 0 ? fs.readSync(fd, buffer, 0, length, position) : 0;
+    return buffer.subarray(0, count);
+}