repoburg 1.3.254 → 1.3.256

package/daemon/dist/tabtape/query/reader.js

@@ -0,0 +1,602 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.View = exports.Reader = exports.RR_META = exports.RR_FULL_SNAPSHOT = void 0;
+exports.parseHead = parseHead;
+/**
+ * Random access to recorded sessions for the features that need more than a
+ * raw stream: MCP tools, reproduction steps, HAR and HTML exports.
+ *
+ * A session is indexed once, line by line, and the index is extended as the
+ * session grows. rrweb lines (most of the bytes) are skipped after reading
+ * their first few bytes; the other records are decoded into compact
+ * summaries and kept in memory. Full records are re-read from disk on demand.
+ */
+const promises_1 = __importDefault(require("fs/promises"));
+const errors_1 = require("../errors");
+const segment_1 = require("../store/segment");
+const types_1 = require("./types");
+/** rrweb event types (rrweb's EventType). */
+exports.RR_FULL_SNAPSHOT = 2;
+exports.RR_META = 4;
+/** How many session indexes are kept in memory. */
+const MAX_CACHED = 8;
+/** Index copies of console text are clipped (bytes); full records are read from disk. */
+const INDEX_TEXT_LIMIT = 8000;
+/** Changed paths kept per state record in the index. */
+const MAX_SUMMARY_PATHS = 20;
+/** Bytes of a line read before deciding whether to keep it. */
+const HEAD_PEEK = 256;
+class Mutex {
+    constructor() {
+        this.tail = Promise.resolve();
+    }
+    run(fn) {
+        const result = this.tail.then(fn);
+        this.tail = result.then(() => undefined, () => undefined);
+        return result;
+    }
+}
+/** Indexes sessions of a store. */
+class Reader {
+    constructor(st) {
+        this.st = st;
+        this.cache = new Map();
+        /** Most recently used last. */
+        this.lru = [];
+    }
+    /** Indexes any new data of the session and returns a view of it. */
+    async view(id) {
+        const { session, segments } = this.st.snapshot(id);
+        const idx = this.index(id);
+        return idx.lock.run(async () => {
+            const present = new Set(segments.map((s) => s.seq));
+            for (const seq of [...idx.segs.keys()]) {
+                if (!present.has(seq))
+                    idx.segs.delete(seq); // evicted
+            }
+            const v = new View(session);
+            for (const sg of segments) {
+                let si = idx.segs.get(sg.seq);
+                if (!si) {
+                    si = { path: sg.path, size: 0, lines: [] };
+                    idx.segs.set(sg.seq, si);
+                }
+                si.path = sg.path; // changes when the segment is compressed
+                if (si.size < sg.size) {
+                    try {
+                        await extend(si, sg.seq, sg.size);
+                    }
+                    catch (err) {
+                        if ((0, errors_1.isNotExist)(err)) {
+                            idx.segs.delete(sg.seq); // evicted after the snapshot
+                            continue;
+                        }
+                        throw new Error(`index segment ${sg.seq}: ${(0, errors_1.errorMessage)(err)}`);
+                    }
+                }
+                v.paths.set(sg.seq, si.path);
+                v.sizes.set(sg.seq, si.size);
+                for (const l of si.lines)
+                    v.lines.push(l);
+            }
+            v.lines.sort((a, b) => a.ts - b.ts); // stable: ties keep file order
+            v.computeRange();
+            return v;
+        });
+    }
+    index(id) {
+        let idx = this.cache.get(id);
+        if (!idx) {
+            idx = { lock: new Mutex(), segs: new Map() };
+            this.cache.set(id, idx);
+        }
+        this.lru = this.lru.filter((s) => s !== id);
+        this.lru.push(id);
+        while (this.lru.length > MAX_CACHED) {
+            this.cache.delete(this.lru[0]);
+            this.lru.shift();
+        }
+        return idx;
+    }
+    /** Drops the cached index of a session (after it was deleted). */
+    forget(id) {
+        this.cache.delete(id);
+        this.lru = this.lru.filter((s) => s !== id);
+    }
+}
+exports.Reader = Reader;
+/**
+ * A consistent picture of one session: every line known when it was built,
+ * ordered like the replay UI orders them (by timestamp, ties in file order).
+ */
+class View {
+    constructor(session) {
+        this.session = session;
+        this.lines = [];
+        /**
+         * The replay's time origin: the first rrweb Meta event that is followed by
+         * a full snapshot, else the first record. Offsets (and the UI's ?t= deep
+         * links) count from here.
+         */
+        this.start = 0;
+        this.end = 0;
+        this.replayable = false;
+        this.paths = new Map();
+        this.sizes = new Map();
+        /** The last compressed segment read by readLine, which cannot seek. */
+        this.gz = null;
+        this.byId = null;
+    }
+    /** @internal */
+    computeRange() {
+        if (this.lines.length === 0)
+            return;
+        const rr = this.lines.filter((l) => l.type === 'rrweb');
+        for (let i = 0; i + 1 < rr.length; i++) {
+            if (rr[i].rr === exports.RR_META && rr[i + 1].rr === exports.RR_FULL_SNAPSHOT) {
+                this.replayable = true;
+                this.start = rr[i].ts;
+                this.end = Math.max(rr[rr.length - 1].ts, this.start);
+                return;
+            }
+        }
+        this.start = this.lines[0].ts;
+        this.end = this.lines[this.lines.length - 1].ts;
+    }
+    /** A timestamp as milliseconds from the replay start, clamped to the replay's range like the UI does. */
+    offset(ts) {
+        return Math.min(Math.max(ts - this.start, 0), this.end - this.start);
+    }
+    /** The replay's length in milliseconds. */
+    duration() {
+        return this.end - this.start;
+    }
+    /** A line's raw bytes (without the newline). */
+    async readLine(l) {
+        let p = this.paths.get(l.seq);
+        if (p === undefined)
+            throw new errors_1.GoneError();
+        if (!p.endsWith('.gz')) {
+            try {
+                return await readAt(p, l.off, l.len);
+            }
+            catch (err) {
+                if (!(0, errors_1.isNotExist)(err))
+                    throw err;
+            }
+            p += '.gz'; // compressed since the view was built
+        }
+        let data;
+        try {
+            data = await this.compressed(l.seq, p);
+        }
+        catch (err) {
+            if ((0, errors_1.isNotExist)(err))
+                throw new errors_1.GoneError();
+            throw err;
+        }
+        const end = l.off + l.len;
+        if (end > data.length)
+            throw new errors_1.GoneError();
+        return Buffer.from(data.subarray(l.off, end));
+    }
+    /** A compressed segment's content, keeping the last one: lines are usually read in order. */
+    compressed(seq, p) {
+        if (this.gz && this.gz.seq === seq)
+            return this.gz.data;
+        const data = (0, segment_1.readSegmentBytes)(p, this.sizes.get(seq) ?? 0);
+        this.gz = { seq, data };
+        data.catch(() => {
+            if (this.gz?.data === data)
+                this.gz = null;
+        });
+        return data;
+    }
+    async decodeData(l) {
+        const b = await this.readLine(l);
+        let v;
+        try {
+            v = JSON.parse(b.toString('utf8'));
+        }
+        catch (err) {
+            throw new Error(`decode record ${l.id}: ${(0, errors_1.errorMessage)(err)}`);
+        }
+        return (0, types_1.isObj)(v) ? v.data : undefined;
+    }
+    /** A network record with headers and bodies. */
+    async fullNetwork(l) {
+        return (0, types_1.decodeNetwork)(await this.decodeData(l));
+    }
+    /** A console record with its arguments and full text. */
+    async fullConsole(l) {
+        return (0, types_1.decodeConsole)(await this.decodeData(l));
+    }
+    /** A WebSocket message with its data. */
+    async fullWS(l) {
+        return (0, types_1.decodeWS)(await this.decodeData(l));
+    }
+    /** A state record with its payload, changes and state. */
+    async fullState(l) {
+        return (0, types_1.decodeState)(await this.decodeData(l));
+    }
+    /** The "data" field of an rrweb event (e.g. the Meta event's width and height). */
+    async rrwebData(l) {
+        const d = await this.decodeData(l);
+        return (0, types_1.isObj)(d) ? d.data : undefined;
+    }
+    /** The line with the given id. */
+    find(id) {
+        if (!this.byId) {
+            this.byId = new Map();
+            for (const l of this.lines)
+                this.byId.set(l.id, l);
+        }
+        return this.byId.get(id);
+    }
+    /**
+     * The last report of each Web Vitals metric of each page load. A metric is
+     * reported again whenever it changes, so every earlier report is
+     * superseded.
+     */
+    finalVitals() {
+        const last = new Map();
+        for (const l of this.lines) {
+            if (l.perf && l.perf.kind === 'vital')
+                last.set(`${l.perf.id}\u0000${l.perf.name}`, l);
+        }
+        return new Set(last.values());
+    }
+    /**
+     * Calls fn with every line of the view and its raw bytes (without the
+     * newline), in file order: segment by segment, oldest first. Segments
+     * deleted meanwhile are skipped.
+     */
+    async scan(fn) {
+        const bySeg = new Map();
+        for (const l of this.lines) {
+            let list = bySeg.get(l.seq);
+            if (!list)
+                bySeg.set(l.seq, (list = []));
+            list.push(l);
+        }
+        const seqs = [...bySeg.keys()].sort((a, b) => a - b);
+        for (const seq of seqs) {
+            const lines = bySeg.get(seq).sort((a, b) => a.off - b.off);
+            await this.scanSegment(seq, lines, fn);
+        }
+    }
+    async scanSegment(seq, lines, fn) {
+        const cursor = new Cursor((0, segment_1.readSegment)(this.paths.get(seq), 0, this.sizes.get(seq) ?? 0));
+        try {
+            let pos = 0;
+            for (const l of lines) {
+                if (l.off > pos) {
+                    if (!(await cursor.skip(l.off - pos)))
+                        return; // truncated meanwhile
+                    pos = l.off;
+                }
+                const b = await cursor.read(l.len + 1);
+                if (!b)
+                    return; // truncated meanwhile (deleted and recreated); stop here
+                pos += l.len + 1;
+                const pending = fn(l, b.subarray(0, l.len));
+                if (pending)
+                    await pending;
+            }
+        }
+        catch (err) {
+            if ((0, errors_1.isNotExist)(err))
+                return; // evicted
+            throw err;
+        }
+        finally {
+            await cursor.close();
+        }
+    }
+}
+exports.View = View;
+/** Sequential reads over a stream of chunks. */
+class Cursor {
+    constructor(gen) {
+        this.parts = [];
+        this.avail = 0;
+        this.done = false;
+        this.it = gen[Symbol.asyncIterator]();
+    }
+    async pull() {
+        if (this.done)
+            return false;
+        const r = await this.it.next();
+        if (r.done) {
+            this.done = true;
+            return false;
+        }
+        this.parts.push(r.value);
+        this.avail += r.value.length;
+        return true;
+    }
+    async skip(n) {
+        while (n > 0) {
+            if (this.avail === 0 && !(await this.pull()))
+                return false;
+            const p = this.parts[0];
+            const k = Math.min(n, p.length);
+            if (k === p.length)
+                this.parts.shift();
+            else
+                this.parts[0] = p.subarray(k);
+            this.avail -= k;
+            n -= k;
+        }
+        return true;
+    }
+    async read(n) {
+        while (this.avail < n) {
+            if (!(await this.pull()))
+                return null;
+        }
+        if (this.parts[0].length >= n) {
+            const out = this.parts[0].subarray(0, n);
+            if (this.parts[0].length === n)
+                this.parts.shift();
+            else
+                this.parts[0] = this.parts[0].subarray(n);
+            this.avail -= n;
+            return out;
+        }
+        const out = Buffer.allocUnsafe(n);
+        let o = 0;
+        while (o < n) {
+            const p = this.parts[0];
+            const k = Math.min(p.length, n - o);
+            p.copy(out, o, 0, k);
+            o += k;
+            if (k === p.length)
+                this.parts.shift();
+            else
+                this.parts[0] = p.subarray(k);
+        }
+        this.avail -= n;
+        return out;
+    }
+    async close() {
+        await this.it.return?.(undefined);
+    }
+}
+async function readAt(p, off, n) {
+    const fh = await promises_1.default.open(p, 'r');
+    try {
+        const b = Buffer.alloc(n);
+        const { bytesRead } = await fh.read(b, 0, n, off);
+        if (bytesRead < n)
+            throw new Error('read record: EOF');
+        return b;
+    }
+    finally {
+        await fh.close();
+    }
+}
+function peek(parts, n) {
+    if (parts.length === 1)
+        return parts[0].subarray(0, n);
+    const out = [];
+    let len = 0;
+    for (const p of parts) {
+        if (len >= n)
+            break;
+        const piece = p.subarray(0, n - len);
+        out.push(piece);
+        len += piece.length;
+    }
+    return Buffer.concat(out, len);
+}
+/** Indexes the segment from si.size up to size. */
+async function extend(si, seq, size) {
+    let off = si.size; // absolute offset of the next unread byte
+    let lineStart = off;
+    let parts = []; // the current line's bytes, while kept
+    let partsLen = 0;
+    // 0: undecided, 1: keep the whole line, 2: skip (an rrweb line whose header is known)
+    let mode = 0;
+    let head = NO_HEAD;
+    for await (const chunk of (0, segment_1.readSegment)(si.path, si.size, size - si.size)) {
+        let pos = 0;
+        while (pos < chunk.length) {
+            const nl = chunk.indexOf(0x0a, pos);
+            const end = nl < 0 ? chunk.length : nl;
+            if (mode !== 2 && end > pos) {
+                parts.push(chunk.subarray(pos, end));
+                partsLen += end - pos;
+            }
+            if (mode === 0 && (partsLen >= HEAD_PEEK || nl >= 0)) {
+                head = parseHead(peek(parts, HEAD_PEEK));
+                if (head.ok && head.type === 'rrweb' && head.rr >= 0) {
+                    mode = 2;
+                    parts = [];
+                    partsLen = 0;
+                }
+                else {
+                    mode = 1;
+                }
+            }
+            off += end - pos;
+            if (nl < 0)
+                break;
+            const l = { id: '', seq, n: si.lines.length, off: lineStart, len: off - lineStart, ts: 0, type: '', rr: -1 };
+            let ok = true;
+            if (mode === 2) {
+                l.ts = head.ts;
+                l.type = head.type;
+                l.rr = head.rr;
+            }
+            else {
+                try {
+                    decodeLine(l, Buffer.concat(parts, partsLen).toString('utf8'));
+                }
+                catch {
+                    ok = false; // not a record; the UI skips it too
+                }
+            }
+            if (ok) {
+                l.id = lineID(l.type, seq, l.n);
+                si.lines.push(l);
+            }
+            off += 1; // the newline
+            pos = nl + 1;
+            lineStart = off;
+            parts = [];
+            partsLen = 0;
+            mode = 0;
+        }
+    }
+    // Sizes only cover complete lines; a missing newline means the data
+    // changed under us: stop at the last complete line.
+    si.size = lineStart;
+}
+const clipBytes = (s, n) => {
+    const b = Buffer.from(s, 'utf8');
+    if (b.length <= n)
+        return s;
+    let cut = n;
+    while (cut > 0 && (b[cut] & 0xc0) === 0x80)
+        cut--;
+    return b.subarray(0, cut).toString('utf8') + '…';
+};
+function decodeLine(l, text) {
+    const v = JSON.parse(text);
+    if (!(0, types_1.isObj)(v))
+        throw new Error('not a record');
+    if (v.ts !== undefined && typeof v.ts !== 'number')
+        throw new Error('bad ts');
+    if (v.type !== undefined && typeof v.type !== 'string')
+        throw new Error('bad type');
+    l.ts = Math.trunc(v.ts ?? 0);
+    l.type = v.type ?? '';
+    const d = v.data;
+    switch (l.type) {
+        case 'rrweb': {
+            const t = (0, types_1.isObj)(d) ? d.type : undefined;
+            if (t !== undefined && typeof t !== 'number')
+                throw new Error('bad rrweb type');
+            l.rr = Math.trunc(t ?? 0);
+            break;
+        }
+        case 'network': {
+            const n = (0, types_1.decodeNetwork)(d);
+            delete n.reqBody;
+            delete n.resBody;
+            delete n.reqHeaders;
+            delete n.resHeaders;
+            l.net = n;
+            break;
+        }
+        case 'console': {
+            const c = (0, types_1.decodeConsole)(d);
+            delete c.args;
+            c.text = clipBytes(c.text, INDEX_TEXT_LIMIT);
+            c.stack = clipBytes(c.stack, INDEX_TEXT_LIMIT);
+            l.con = c;
+            break;
+        }
+        case 'event':
+            l.ev = (0, types_1.decodeEvent)(d);
+            break;
+        case 'perf':
+            l.perf = (0, types_1.decodePerf)(d);
+            break;
+        case 'ws': {
+            const f = (0, types_1.decodeWS)(d);
+            delete f.data;
+            l.ws = f;
+            break;
+        }
+        case 'state': {
+            const s = (0, types_1.decodeState)(d);
+            const changes = s.changes ?? [];
+            l.state = {
+                store: s.store,
+                kind: s.kind,
+                action: s.action,
+                paths: changes.slice(0, MAX_SUMMARY_PATHS).map((c) => c.path),
+                changes: changes.length,
+                full: s.state !== undefined,
+                burst: s.burst,
+                reason: s.reason,
+            };
+            break;
+        }
+    }
+}
+const PREFIXES = {
+    network: 'n',
+    console: 'c',
+    event: 'e',
+    rrweb: 'r',
+    perf: 'p',
+    ws: 'w',
+    state: 's',
+};
+function lineID(type, seq, n) {
+    const prefix = Object.prototype.hasOwnProperty.call(PREFIXES, type) ? PREFIXES[type] : 'x';
+    return `${prefix}${seq}-${n}`;
+}
+const NO_HEAD = { ok: false, ts: 0, type: '', rr: -1 };
+const SID_PREFIX = Buffer.from('{"sid":"');
+const TS_PREFIX = Buffer.from(',"ts":');
+const TYPE_PREFIX = Buffer.from('"type":"');
+const RR_PREFIX = Buffer.from(',"data":{"type":');
+const NUMBER = /^-?\d+(\.\d+)?([eE][+-]?\d+)?$/;
+function startsWith(b, prefix) {
+    return b.length >= prefix.length && b.subarray(0, prefix.length).equals(prefix);
+}
+/**
+ * Reads the start of a line as the store writes it:
+ *
+ *   {"sid":"…","ts":123,"type":"rrweb","data":{"type":4,…
+ *
+ * ok is false for any other layout; callers then decode the line. rr is -1
+ * when the line is not rrweb or its event type is not where expected.
+ */
+function parseHead(b) {
+    if (!startsWith(b, SID_PREFIX))
+        return NO_HEAD;
+    let rest = b.subarray(SID_PREFIX.length);
+    let i = rest.indexOf(0x22); // "
+    if (i < 0)
+        return NO_HEAD;
+    rest = rest.subarray(i + 1);
+    if (!startsWith(rest, TS_PREFIX))
+        return NO_HEAD;
+    rest = rest.subarray(TS_PREFIX.length);
+    i = rest.indexOf(0x2c); // ,
+    if (i < 0)
+        return NO_HEAD;
+    const tsText = rest.subarray(0, i).toString('latin1');
+    if (!NUMBER.test(tsText))
+        return NO_HEAD;
+    const ts = Math.trunc(Number(tsText));
+    rest = rest.subarray(i + 1);
+    if (!startsWith(rest, TYPE_PREFIX))
+        return NO_HEAD;
+    rest = rest.subarray(TYPE_PREFIX.length);
+    i = rest.indexOf(0x22);
+    if (i < 0)
+        return NO_HEAD;
+    const type = rest.subarray(0, i).toString('utf8');
+    if (type !== 'rrweb')
+        return { ok: true, ts, type, rr: -1 };
+    rest = rest.subarray(i + 1);
+    let rr = -1;
+    if (startsWith(rest, RR_PREFIX)) {
+        rest = rest.subarray(RR_PREFIX.length);
+        let j = 0;
+        while (j < rest.length && rest[j] >= 0x30 && rest[j] <= 0x39)
+            j++;
+        if (j > 0 && j < rest.length)
+            rr = Number(rest.subarray(0, j).toString('latin1'));
+    }
+    return { ok: true, ts, type, rr };
+}