repoburg 1.3.255 → 1.3.256

package/daemon/dist/tabtape/har/har.js

@@ -0,0 +1,582 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildHar = buildHar;
+/**
+ * Converts a recorded session's network data to HAR 1.2, the format Chrome
+ * DevTools, Charles and Proxyman import.
+ *
+ * Requests captured in the page (fetch, XHR, beacons) carry the headers and
+ * bodies the page could see and two timings: waiting for the response
+ * headers, and reading the body. Assets seen through Resource Timing have a
+ * size and a duration only. Requests recorded in deep mode (through the
+ * Chrome DevTools Protocol) have the real headers, the full timing split
+ * (DNS, connect, TLS, ...), the protocol and the server address, and
+ * WebSocket connections carry their messages in _webSocketMessages, like
+ * Chrome's own HAR export. Unknown values are -1, as the format asks.
+ *
+ * Objects are built with the fields in HAR order and optional fields left
+ * out when empty, so JSON.stringify(har, null, 2) writes the same file the Go
+ * server did.
+ */
+const network_1 = require("../query/network");
+const types_1 = require("../query/types");
+const gostd_1 = require("./gostd");
+const UNKNOWN = -1;
+/** Converts the session's page views, network records and WebSocket messages. */
+async function buildHar(v, opts) {
+    const pages = [];
+    const nets = [];
+    const wsLines = new Set();
+    for (const l of v.lines) {
+        if (l.ev && l.ev.name === 'page_view')
+            pages.push({ ts: l.ts, url: l.ev.url, title: l.ev.title });
+        else if (l.net && inRange(opts, v, l.ts))
+            nets.push(l);
+        else if (l.ws && l.ws.error === '' && inRange(opts, v, l.ts))
+            wsLines.add(l);
+    }
+    // Headers, bodies and message data are not in the index: read the
+    // records, in file order, and keep the ones in range.
+    const want = new Set(nets);
+    const full = new Map();
+    const messages = new Map();
+    await v.scan((l, raw) => {
+        if (want.has(l)) {
+            const data = recordData(raw);
+            if (data === UNREADABLE)
+                return; // the index decoded it; a changed file is skipped
+            full.set(l, (0, types_1.decodeNetwork)(data));
+        }
+        else if (wsLines.has(l)) {
+            const data = recordData(raw);
+            if (data === UNREADABLE)
+                return;
+            const f = (0, types_1.decodeWS)(data);
+            let list = messages.get(f.conn);
+            if (!list)
+                messages.set(f.conn, (list = []));
+            list.push({ ts: l.ts, f });
+        }
+    });
+    const entries = [];
+    const used = new Set();
+    let deep = false;
+    let inPage = false;
+    for (const l of nets) {
+        const n = full.get(l);
+        if (!n)
+            continue; // deleted by the ring buffer meanwhile
+        deep = deep || n.source === 'cdp';
+        inPage = inPage || n.source !== 'cdp';
+        const i = pageFor(pages, l.ts, n.kind === 'navigation');
+        if (i >= 0)
+            used.add(i);
+        const ws = (0, types_1.isWebSocket)(n) ? wsMessages(messages.get(n.id)) : [];
+        entries.push(entry(opts, l.ts, n, pageURL(pages, l.ts), i >= 0 ? pageID(i) : '', ws));
+    }
+    // Stable, like Go's slices.SortStableFunc; the times are ASCII.
+    entries.sort((a, b) => (a.startedDateTime < b.startedDateTime ? -1 : a.startedDateTime > b.startedDateTime ? 1 : 0));
+    const harPages = [];
+    pages.forEach((p, i) => {
+        if (!used.has(i) && !inRange(opts, v, p.ts))
+            return;
+        harPages.push({
+            startedDateTime: isoTime(p.ts),
+            id: pageID(i),
+            title: p.title || p.url,
+            pageTimings: { onContentLoad: UNKNOWN, onLoad: UNKNOWN },
+        });
+    });
+    const b = browser(v.session.userAgent ?? '');
+    return {
+        log: {
+            version: '1.2',
+            creator: { name: 'tabtape', version: opts.version || 'dev' },
+            ...(b ? { browser: b } : {}),
+            pages: harPages,
+            entries,
+            comment: comment(opts, deep, inPage),
+        },
+    };
+}
+const UNREADABLE = Symbol('unreadable');
+/** A raw record's "data" field; UNREADABLE when the line is not a record object (it changed on disk). */
+function recordData(raw) {
+    let rec;
+    try {
+        rec = JSON.parse(raw.toString('utf8'));
+    }
+    catch {
+        return UNREADABLE;
+    }
+    if (!(0, types_1.isObj)(rec))
+        return UNREADABLE;
+    const data = rec.data;
+    return data === undefined || data === null || (0, types_1.isObj)(data) ? data : UNREADABLE;
+}
+/**
+ * Uses the replay's clamped offsets, like the UI: requests before the first
+ * snapshot count as 0.
+ */
+function inRange(o, v, ts) {
+    const off = v.offset(ts);
+    if (off < (o.from ?? 0))
+        return false;
+    const to = o.to ?? 0;
+    return to <= 0 || off <= to;
+}
+function pageID(i) {
+    return `page_${i + 1}`;
+}
+/**
+ * The page a request belongs to: the last page view at or before it, or for
+ * a document request the page it loads. -1 means none.
+ */
+function pageFor(pages, ts, document) {
+    // Binary search for i: pages[:i] start at or before ts.
+    let i = 0;
+    let j = pages.length;
+    while (i < j) {
+        const h = (i + j) >>> 1;
+        if (pages[h].ts <= ts)
+            i = h + 1;
+        else
+            j = h;
+    }
+    if (document && i < pages.length)
+        return i;
+    return i - 1;
+}
+function pageURL(pages, ts) {
+    const i = pageFor(pages, ts, false);
+    return i >= 0 ? pages[i].url : '';
+}
+function entry(o, ts, n, page, pageref, ws) {
+    const u = absolute(n.url, page);
+    const version = httpVersion(n.protocol);
+    const duration = n.duration !== undefined ? Math.max(n.duration, 0) : 0;
+    let timings = { blocked: UNKNOWN, dns: UNKNOWN, connect: UNKNOWN, send: 0, wait: 0, receive: 0, ssl: UNKNOWN };
+    let time;
+    if ((0, types_1.isWebSocket)(n)) {
+        // The connection's lifetime, so viewers draw it across the waterfall.
+        timings.receive = duration;
+        time = duration;
+    }
+    else if (n.timing) {
+        [timings, time] = deepTimings(n.timing);
+    }
+    else {
+        timings.wait = duration;
+        if (n.ttfb !== undefined && n.ttfb >= 0 && n.ttfb <= duration) {
+            timings.wait = n.ttfb;
+            timings.receive = duration - n.ttfb;
+        }
+        time = duration;
+    }
+    let status = n.status;
+    let error = '';
+    const notes = [];
+    if (n.error !== '' || n.corsErrorStatus !== undefined || n.blockedReason !== '') {
+        status = 0;
+        error = n.error || (0, network_1.failureReason)(n);
+        notes.push('Failed: ' + (0, network_1.failureReason)(n));
+    }
+    else if (n.pending && (0, types_1.isWebSocket)(n)) {
+        notes.push('Still open when the recording stopped');
+    }
+    else if (n.pending) {
+        notes.push('Still in flight when the page unloaded or the recording stopped');
+    }
+    if ((0, types_1.isAsset)(n) && n.source !== 'cdp')
+        notes.push('Seen through Resource Timing: no headers or body');
+    if (n.worker)
+        notes.push(`Requested by a ${n.worker.type.replace(/_/g, ' ')}: ${n.worker.url}`);
+    for (const c of n.blockedCookies ?? [])
+        notes.push(`Cookie ${c.name} not sent: ${(0, network_1.cookieReasons)(c.reasons)}`);
+    for (const c of n.blockedSetCookies ?? [])
+        notes.push(`Set-Cookie ${c.name} rejected: ${(0, network_1.cookieReasons)(c.reasons)}`);
+    const note = notes.join('. ');
+    const [reqHeaders, reqCookies] = headers(o, n.reqHeaders, 'cookie');
+    const [resHeaders, resCookies] = headers(o, n.resHeaders, 'set-cookie');
+    let redirectURL = '';
+    const loc = header(n.resHeaders, 'location');
+    if (loc !== '' && n.status >= 300 && n.status < 400)
+        redirectURL = absolute(loc, u);
+    let postData;
+    let bodySize = 0;
+    if (n.reqBody !== undefined || n.reqBodyNote !== '') {
+        const mimeType = header(n.reqHeaders, 'content-type') || 'application/octet-stream';
+        if (n.reqBody !== undefined) {
+            bodySize = Buffer.byteLength(n.reqBody, 'utf8');
+            let pdComment = '';
+            if (n.reqBodyTruncated) {
+                pdComment = truncated(n.reqBody);
+                bodySize = UNKNOWN;
+            }
+            postData = { mimeType, text: n.reqBody, ...(pdComment ? { comment: pdComment } : {}) };
+        }
+        else {
+            postData = { mimeType, text: '', comment: n.reqBodyNote };
+            bodySize = UNKNOWN;
+        }
+    }
+    let size = 0;
+    if (n.resSize !== undefined)
+        size = intOf(n.resSize);
+    else if (n.resBody !== undefined && !n.resBodyTruncated)
+        size = Buffer.byteLength(n.resBody, 'utf8');
+    let text = '';
+    let contentComment = '';
+    if (n.resBody !== undefined) {
+        text = n.resBody;
+        if (n.resBodyTruncated)
+            contentComment = truncated(n.resBody);
+    }
+    else if (n.resBodyNote !== '') {
+        contentComment = n.resBodyNote;
+    }
+    const content = {
+        size,
+        mimeType: n.mime || header(n.resHeaders, 'content-type') || 'x-unknown',
+        ...(text ? { text } : {}),
+        ...(contentComment ? { comment: contentComment } : {}),
+    };
+    const serverIP = n.remoteAddress !== '' ? (0, network_1.remoteIP)(n.remoteAddress) : '';
+    const fromCache = n.cache === 'memory' || n.cache === 'disk' ? n.cache : '';
+    const resourceType = resourceTypeOf(n);
+    return {
+        ...(pageref ? { pageref } : {}),
+        startedDateTime: isoTime(ts),
+        time,
+        request: {
+            method: n.method.toUpperCase() || 'GET',
+            url: u,
+            httpVersion: version,
+            cookies: reqCookies,
+            headers: reqHeaders,
+            queryString: queryString(u),
+            ...(postData ? { postData } : {}),
+            headersSize: UNKNOWN,
+            bodySize,
+        },
+        response: {
+            status,
+            statusText: n.statusText,
+            httpVersion: version,
+            cookies: resCookies,
+            headers: resHeaders,
+            content,
+            redirectURL,
+            headersSize: UNKNOWN,
+            bodySize: UNKNOWN,
+            ...(n.transferSize !== undefined ? { _transferSize: intOf(n.transferSize) } : {}),
+            ...(error ? { _error: error } : {}),
+        },
+        cache: {},
+        timings,
+        ...(serverIP ? { serverIPAddress: serverIP } : {}),
+        ...(note ? { comment: note } : {}),
+        ...(resourceType ? { _resourceType: resourceType } : {}),
+        ...(fromCache ? { _fromCache: fromCache } : {}),
+        ...(ws.length > 0 ? { _webSocketMessages: ws } : {}),
+    };
+}
+function truncated(body) {
+    return `Truncated by tabtape after ${runeCount(body)} characters`;
+}
+/**
+ * Converts a deep mode timing split; HAR wants send, wait and receive
+ * non-negative and time to be the sum of the known phases (ssl is part of
+ * connect).
+ */
+function deepTimings(t) {
+    const known = (v) => (v === undefined || v < 0 ? UNKNOWN : v);
+    const nonNeg = (v) => Math.max(known(v), 0);
+    const tm = {
+        blocked: known(t.blocked),
+        dns: known(t.dns),
+        connect: known(t.connect),
+        send: nonNeg(t.send),
+        wait: nonNeg(t.wait),
+        receive: nonNeg(t.receive),
+        ssl: known(t.ssl),
+    };
+    let total = tm.send + tm.wait + tm.receive;
+    for (const v of [tm.blocked, tm.dns, tm.connect])
+        total += Math.max(v, 0);
+    return [tm, total];
+}
+/**
+ * Lists a connection's recorded messages; binary data and data left out by
+ * the recorder's limits are empty.
+ */
+function wsMessages(ms) {
+    if (!ms || ms.length === 0)
+        return [];
+    ms.sort((a, b) => a.ts - b.ts); // stable
+    return ms.map((m) => ({ type: m.f.dir === 'send' ? 'send' : 'receive', time: m.ts / 1000, opcode: m.f.opcode, data: m.f.data ?? '' }));
+}
+/** Names a protocol like Chrome's HAR export does. */
+function httpVersion(protocol) {
+    const p = protocol.toLowerCase();
+    if (p === '')
+        return '';
+    if (p === 'h2')
+        return 'HTTP/2';
+    if (p.startsWith('h3'))
+        return 'HTTP/3';
+    if (p.startsWith('http/'))
+        return p.toUpperCase();
+    return protocol;
+}
+/**
+ * Lists headers sorted by name, leaving out sensitive ones unless secrets is
+ * set, and parses the cookie header given.
+ */
+function headers(o, m, cookieHeader) {
+    const out = [];
+    const cookies = [];
+    if (!m)
+        return [out, cookies];
+    for (const name of sortedKeys(m)) {
+        if (!o.secrets && o.redact.has(name))
+            continue;
+        out.push({ name, value: m[name] });
+        if (name.toLowerCase() === cookieHeader) {
+            for (const c of parseCookies(m[name], cookieHeader === 'set-cookie'))
+                cookies.push(c);
+        }
+    }
+    return [out, cookies];
+}
+function parseCookies(v, set) {
+    if (set) {
+        // One cookie per line: "name=value; Path=/; HttpOnly".
+        const out = [];
+        for (const line of v.split('\n')) {
+            const c = (0, gostd_1.parseSetCookie)((0, gostd_1.trimSpace)(line));
+            if (c)
+                out.push(c);
+        }
+        return out;
+    }
+    return (0, gostd_1.parseCookie)(v) ?? [];
+}
+function header(m, name) {
+    if (!m)
+        return '';
+    for (const k of Object.keys(m)) {
+        if (k.toLowerCase() === name)
+            return m[k];
+    }
+    return '';
+}
+function queryString(raw) {
+    const out = [];
+    const u = (0, gostd_1.parseURL)(raw);
+    if (!u || u.rawQuery === '')
+        return out;
+    for (const part of u.rawQuery.split('&')) {
+        if (part === '')
+            continue;
+        const eq = part.indexOf('=');
+        const k = eq < 0 ? part : part.slice(0, eq);
+        const val = eq < 0 ? '' : part.slice(eq + 1);
+        const name = (0, gostd_1.queryUnescape)(k);
+        const value = (0, gostd_1.queryUnescape)(val);
+        out.push(name === null || value === null ? { name: k, value: val } : { name, value });
+    }
+    return out;
+}
+/**
+ * Resolves a URL against the page it was requested from. The resolved URL is
+ * the WHATWG serialization (Go wrote its own, which differs only in
+ * normalization details such as escaping spaces in the query).
+ */
+function absolute(raw, base) {
+    const u = (0, gostd_1.parseURL)(raw);
+    if (!u || u.scheme !== '')
+        return raw;
+    const b = (0, gostd_1.parseURL)(base);
+    if (!b || b.scheme === '')
+        return raw;
+    try {
+        return new URL(raw, base).href;
+    }
+    catch {
+        return raw;
+    }
+}
+/** Maps a record to Chrome's resource types. */
+function resourceTypeOf(n) {
+    if (n.resourceType !== '')
+        return n.resourceType; // deep mode: Chrome's own
+    switch (n.kind) {
+        case 'websocket':
+            return 'websocket';
+        case 'fetch':
+            return 'fetch';
+        case 'xhr':
+            return 'xhr';
+        case 'beacon':
+            return 'ping';
+        case 'navigation':
+            return 'document';
+    }
+    const t = typeOfMime(n.mime);
+    if (t !== '')
+        return t;
+    switch ((0, gostd_1.pathExt)(urlPath(n.url)).toLowerCase()) {
+        case '.js':
+        case '.mjs':
+            return 'script';
+        case '.css':
+            return 'stylesheet';
+        case '.png':
+        case '.jpg':
+        case '.jpeg':
+        case '.gif':
+        case '.svg':
+        case '.webp':
+        case '.avif':
+        case '.ico':
+        case '.bmp':
+            return 'image';
+        case '.woff':
+        case '.woff2':
+        case '.ttf':
+        case '.otf':
+        case '.eot':
+            return 'font';
+        case '.mp4':
+        case '.webm':
+        case '.mp3':
+        case '.ogg':
+        case '.wav':
+        case '.m4a':
+            return 'media';
+    }
+    switch (n.initiator) {
+        case 'script':
+            return 'script';
+        case 'img':
+        case 'image':
+            return 'image';
+        case 'css':
+            return 'stylesheet';
+        case 'xmlhttprequest':
+            return 'xhr';
+        case 'fetch':
+            return 'fetch';
+        case 'beacon':
+            return 'ping';
+        case 'iframe':
+        case 'frame':
+            return 'document';
+        case 'video':
+        case 'audio':
+            return 'media';
+    }
+    return 'other';
+}
+function typeOfMime(m) {
+    const mt = (0, gostd_1.mediaType)(m);
+    if (mt === null)
+        return '';
+    if (mt === 'text/css')
+        return 'stylesheet';
+    if (mt.indexOf('javascript') >= 0)
+        return 'script';
+    if (mt.startsWith('image/'))
+        return 'image';
+    if (mt.startsWith('font/'))
+        return 'font';
+    if (mt.startsWith('video/') || mt.startsWith('audio/'))
+        return 'media';
+    if (mt === 'text/html')
+        return 'document';
+    return '';
+}
+function urlPath(raw) {
+    const u = (0, gostd_1.parseURL)(raw);
+    return u ? u.path : raw;
+}
+const BROWSERS = [
+    { name: 'Microsoft Edge', re: /Edg\/([\d.]+)/ },
+    { name: 'Chrome', re: /Chrome\/([\d.]+)/ },
+    { name: 'Firefox', re: /Firefox\/([\d.]+)/ },
+];
+function browser(ua) {
+    for (const b of BROWSERS) {
+        const m = b.re.exec(ua);
+        if (m)
+            return { name: b.name, version: m[1] };
+    }
+    const safari = safariVersion(ua);
+    return safari !== null ? { name: 'Safari', version: safari } : null;
+}
+/**
+ * Go's `Version/([\d.]+).*Safari/` without backtracking: the first
+ * "Version/<digits>" followed by "Safari/" on the same line.
+ */
+function safariVersion(ua) {
+    const re = /Version\/([\d.]+)/g;
+    for (let m = re.exec(ua); m; m = re.exec(ua)) {
+        const end = m.index + m[0].length;
+        const s = ua.indexOf('Safari/', end);
+        if (s < 0)
+            return null;
+        const nl = ua.indexOf('\n', end);
+        if (nl < 0 || s < nl)
+            return m[1];
+    }
+    return null;
+}
+const PAGE_NOTE = 'Recorded in the page by tabtape: requests the page made itself (fetch, XHR, beacons) with the headers and bodies it could see, ' +
+    'and assets from Resource Timing. Timings are split into waiting and receiving only.';
+const CDP_NOTE = 'Recorded by tabtape in deep mode, through the Chrome DevTools Protocol: every request of the tab with the headers Chrome sent and received, ' +
+    'the full timing split, and WebSocket messages (_webSocketMessages). Response bodies are kept for fetch and XHR only.';
+const MIXED_NOTE = ' Deep mode ended during the recording: later requests were captured in the page, with the headers the page could see and ' +
+    'timings split into waiting and receiving only.';
+function comment(o, deep, inPage) {
+    let s = PAGE_NOTE;
+    if (deep && inPage)
+        s = CDP_NOTE + MIXED_NOTE;
+    else if (deep)
+        s = CDP_NOTE;
+    const names = o.redact.list();
+    if (!o.secrets && names.length > 0)
+        s += ` Sensitive headers were left out: ${names.join(', ')}.`;
+    return s;
+}
+/** Date's range; timestamps beyond it (never recorded in practice) are clamped rather than throwing. */
+const MAX_DATE_MS = 8.64e15;
+function isoTime(ms) {
+    return new Date(Math.min(Math.max(ms, -MAX_DATE_MS), MAX_DATE_MS)).toISOString();
+}
+/** A non-negative size rounded half away from zero (Go's math.Round); 0 for NaN and negatives. */
+function intOf(f) {
+    if (Number.isNaN(f) || f < 0)
+        return 0;
+    return Math.round(f);
+}
+/** Characters as Go counts runes: code points, a lone surrogate counting as one. */
+function runeCount(s) {
+    let n = 0;
+    for (let i = 0; i < s.length; i++, n++) {
+        const c = s.charCodeAt(i);
+        if (c >= 0xd800 && c <= 0xdbff && i + 1 < s.length) {
+            const d = s.charCodeAt(i + 1);
+            if (d >= 0xdc00 && d <= 0xdfff)
+                i++;
+        }
+    }
+    return n;
+}
+/** Header names sorted case-insensitively; names equal but for case keep a fixed order. */
+function sortedKeys(m) {
+    return Object.keys(m)
+        .map((k) => ({ k, lower: k.toLowerCase() }))
+        .sort((a, b) => (0, gostd_1.compareStrings)(a.lower, b.lower) || (0, gostd_1.compareStrings)(a.k, b.k))
+        .map((x) => x.k);
+}