reverseport 0.0.0-stage → 1.0.0

package/dist/browser/CdpCaptureEvents.js

@@ -0,0 +1,199 @@
+import { CdpCaptureCompleteness } from "./CdpCaptureCompleteness.js";
+import * as bodyShapes from "./CdpCaptureEventBodyShapes.js";
+import * as handlers from "./CdpCaptureEventHandlers.js";
+import { isJsonMediaType } from "./CdpCaptureEventHelpers.js";
+import { cdpStringValue, recordValue } from "./CdpCaptureValues.js";
+/** Event accumulator that validates and normalizes payloads at ingestion. */
+export class CdpCaptureEvents {
+    #scriptMetadataBudgetExclusionsRecorded = false;
+    #networkRequestOccurrences = new Map();
+    #responseBodyExaminedOccurrences = new Map();
+    scripts = new Map();
+    scriptMetadataBudgetOmissionsById = new Map();
+    rejectedScriptMetadataBudgetCount = 0;
+    rejectedSourceMapMetadataBudgetCount = 0;
+    scriptMetadataBytesById = new Map();
+    retainedScriptMetadataBytes = 0;
+    executionContextFrames = new Map();
+    network = new Map();
+    networkRequestTimestamps = new Map();
+    malformedRedirectRequestIds = new Set();
+    webSocketConnectionsById = new Map();
+    console = [];
+    responseMetadata = [];
+    agentHints = [];
+    completeness = new CdpCaptureCompleteness([
+        "network_requests",
+        "console_events",
+        "websocket_connections",
+        "websocket_frames",
+        "metadata",
+    ]);
+    originViolation = false;
+    navigationDuringCapture = false;
+    mainFrameId = undefined;
+    committedDocument = undefined;
+    input;
+    allowedOrigins;
+    constructor(input, allowedOrigins) {
+        this.input = input;
+        this.allowedOrigins = allowedOrigins;
+    }
+    beginAuthorizedFrame(frameId) {
+        this.mainFrameId = frameId;
+        this.originViolation = false;
+        this.navigationDuringCapture = false;
+        this.committedDocument = undefined;
+        this.scripts.clear();
+        this.scriptMetadataBudgetOmissionsById.clear();
+        this.rejectedScriptMetadataBudgetCount = 0;
+        this.rejectedSourceMapMetadataBudgetCount = 0;
+        this.scriptMetadataBytesById.clear();
+        this.retainedScriptMetadataBytes = 0;
+        this.#scriptMetadataBudgetExclusionsRecorded = false;
+        this.executionContextFrames.clear();
+        this.network.clear();
+        this.networkRequestTimestamps.clear();
+        this.#networkRequestOccurrences.clear();
+        this.#responseBodyExaminedOccurrences.clear();
+        this.malformedRedirectRequestIds.clear();
+        this.webSocketConnectionsById.clear();
+        this.console.length = 0;
+        this.responseMetadata.length = 0;
+        this.agentHints.length = 0;
+        this.completeness.reset();
+        if (!this.input.include_console_text)
+            this.completeness.exclude("console_text", "not_approved", null);
+        if (!this.input.include_json_body_shapes)
+            this.completeness.exclude("json_body_shapes", "not_approved", null);
+        if (!this.input.include_websocket_shapes)
+            this.completeness.exclude("websocket_shapes", "not_approved", null);
+    }
+    beginFinalCapture(frameId) {
+        this.mainFrameId = frameId;
+        // Deliberately does not clear `navigationDuringCapture`. A main-frame
+        // navigation observed during the observation window means the page never
+        // settled, and clearing the flag here reported the capture as clean while
+        // carrying pre-navigation scripts, network and console into the inventory
+        // of the new document. The caller now reports `target_changed` instead.
+        // `committedDocument` is dropped so the new document's frame events are
+        // compared against itself rather than the document we just left.
+        this.committedDocument = undefined;
+    }
+    /** Record final script metadata omissions after all scriptParsed replacements. */
+    recordScriptMetadataBudgetExclusions() {
+        if (this.#scriptMetadataBudgetExclusionsRecorded)
+            return;
+        this.#scriptMetadataBudgetExclusionsRecorded = true;
+        const omittedScripts = this.rejectedScriptMetadataBudgetCount;
+        const omittedSourceMaps = this.rejectedSourceMapMetadataBudgetCount +
+            this.scriptMetadataBudgetOmissionsById.size;
+        if (omittedScripts > 0)
+            this.completeness.exclude("scripts", "resource_budget_exhausted", omittedScripts);
+        if (omittedSourceMaps > 0)
+            this.completeness.exclude("source_maps", "resource_budget_exhausted", omittedSourceMaps);
+    }
+    ingest(event) {
+        const params = recordValue(event.params);
+        if (params === undefined)
+            return;
+        switch (event.method) {
+            case "Runtime.executionContextCreated":
+                handlers.handleExecutionContextCreated(this, params);
+                break;
+            case "Runtime.executionContextDestroyed":
+                handlers.handleExecutionContextDestroyed(this, params);
+                break;
+            case "Runtime.executionContextsCleared":
+                this.executionContextFrames.clear();
+                break;
+            case "Debugger.scriptParsed":
+                handlers.handleScriptParsed(this, params);
+                break;
+            case "Network.requestWillBeSent": {
+                const requestId = cdpStringValue(params.requestId);
+                const previous = requestId === undefined ? undefined : this.network.get(requestId);
+                handlers.handleRequestWillBeSent(this, params);
+                if (requestId !== undefined && this.network.get(requestId) !== previous)
+                    this.#networkRequestOccurrences.set(requestId, (this.#networkRequestOccurrences.get(requestId) ?? 0) + 1);
+                break;
+            }
+            case "Network.responseReceived":
+                handlers.handleResponseReceived(this, params);
+                break;
+            case "Network.loadingFinished":
+                handlers.handleLoadingFinished(this, params);
+                break;
+            case "Runtime.consoleAPICalled":
+                handlers.handleConsoleAPICalled(this, params);
+                break;
+            case "Page.frameNavigated":
+                handlers.handleFrameNavigated(this, params);
+                break;
+            case "Network.webSocketCreated":
+                handlers.handleWebSocketCreated(this, params);
+                break;
+            case "Network.webSocketFrameSent":
+                handlers.handleWebSocketFrame(this, params, "sent");
+                break;
+            case "Network.webSocketFrameReceived":
+                handlers.handleWebSocketFrame(this, params, "received");
+                break;
+        }
+    }
+    /** Resolve a transient execution context to one already authorized frame. */
+    frameForScript(script, allowedFrameIds) {
+        return script.frameId !== null && allowedFrameIds.has(script.frameId)
+            ? script.frameId
+            : null;
+    }
+    responseBodyRequests() {
+        if (!this.input.include_json_body_shapes)
+            return [];
+        return [...this.network.values()]
+            .filter((request) => !this.malformedRedirectRequestIds.has(request.request_id))
+            .filter((request) => isJsonMediaType(request.mime_type))
+            .flatMap((request) => {
+            const occurrence = this.#networkRequestOccurrences.get(request.request_id);
+            return occurrence === undefined
+                ? []
+                : [{ requestId: request.request_id, occurrence }];
+        });
+    }
+    currentResponseBodyRequest(candidate) {
+        if (candidate.occurrence !==
+            this.#networkRequestOccurrences.get(candidate.requestId))
+            return undefined;
+        return this.network.get(candidate.requestId);
+    }
+    responseBodyUnavailable(candidate) {
+        if (this.currentResponseBodyRequest(candidate) === undefined)
+            return;
+        this.#responseBodyExaminedOccurrences.set(candidate.requestId, candidate.occurrence);
+        if (this.malformedRedirectRequestIds.has(candidate.requestId))
+            return;
+        bodyShapes.updateResponseBodyShape(this, candidate.requestId, null);
+        this.completeness.unavailable("json_body_shapes");
+    }
+    /**
+     * Mark eligible response bodies that arrived after the fetch pass was
+     * enumerated.
+     */
+    settleUnexaminedResponseBodies() {
+        for (const candidate of this.responseBodyRequests()) {
+            const request = this.currentResponseBodyRequest(candidate);
+            if (request?.body_shapes.response === null &&
+                this.#responseBodyExaminedOccurrences.get(candidate.requestId) !==
+                    candidate.occurrence)
+                this.responseBodyUnavailable(candidate);
+        }
+    }
+    ingestResponseBody(candidate, value) {
+        if (this.currentResponseBodyRequest(candidate) === undefined)
+            return;
+        this.#responseBodyExaminedOccurrences.set(candidate.requestId, candidate.occurrence);
+        if (this.malformedRedirectRequestIds.has(candidate.requestId))
+            return;
+        bodyShapes.ingestResponseBodyShape(this, candidate.requestId, value);
+    }
+}

package/dist/browser/CdpCaptureStorage.js

@@ -0,0 +1,86 @@
+import { CdpConnection } from "./CdpConnection.js";
+import { optionalCdpCommand } from "./CdpOptionalCommand.js";
+import { captureStorageFingerprints, } from "./CdpCaptureStorageFingerprints.js";
+import { numberValue, recordValue, recordsValue, cdpStringValue, } from "./CdpCaptureValues.js";
+/** Capture redacted storage metadata for an authorized page and its origin. */
+export const captureStorage = async (context, pageUrl, limitations) => {
+    const origin = new URL(pageUrl).origin;
+    const quota = recordValue(await optionalCdpCommand(context, "Storage.getUsageAndQuota", { origin }, limitations));
+    const local = context.input.include_storage_keys
+        ? await storageItems(context, origin, true, limitations)
+        : emptyStorageItems;
+    const session = context.input.include_storage_keys
+        ? await storageItems(context, origin, false, limitations)
+        : emptyStorageItems;
+    const indexedRaw = context.input.include_storage_keys
+        ? await optionalCdpCommand(context, "IndexedDB.requestDatabaseNames", { securityOrigin: origin }, limitations)
+        : undefined;
+    const indexedValues = recordValue(indexedRaw)?.databaseNames;
+    const indexed = stringArray(indexedValues);
+    const cacheRaw = context.input.include_storage_keys
+        ? await optionalCdpCommand(context, "CacheStorage.requestCacheNames", { securityOrigin: origin }, limitations)
+        : undefined;
+    const cacheValues = recordsValue(recordValue(cacheRaw)?.caches);
+    const caches = capturedCaches(cacheValues);
+    const fingerprints = context.input.include_storage_fingerprints
+        ? await captureStorageFingerprints({
+            context,
+            origin,
+            cookieUrl: pageUrl,
+            local,
+            session,
+            indexedDbNames: indexed,
+            caches,
+            limitations,
+        })
+        : { items: [], complete: false };
+    const structuredStorageComplete = indexedRaw !== undefined &&
+        Array.isArray(indexedValues) &&
+        indexed.length === indexedValues.length &&
+        cacheRaw !== undefined &&
+        caches.length === cacheValues.length;
+    return {
+        value: {
+            origin,
+            usage_bytes: numberValue(quota?.usage) ?? null,
+            quota_bytes: numberValue(quota?.quota) ?? null,
+            local_storage_keys: local.items.map(({ key }) => key),
+            session_storage_keys: session.items.map(({ key }) => key),
+            indexed_db_names: [...indexed],
+            cache_names: caches.map(({ name }) => name),
+            content_fingerprints: [...fingerprints.items],
+            fingerprint_algorithm: "sha256",
+            fingerprints_complete: fingerprints.complete && structuredStorageComplete,
+            values_redacted: true,
+        },
+    };
+};
+const emptyStorageItems = {
+    items: [],
+    complete: false,
+};
+const capturedCaches = (values) => values.flatMap((cache) => {
+    const name = cdpStringValue(cache.cacheName);
+    const id = cdpStringValue(cache.cacheId);
+    return name === undefined || id === undefined ? [] : [{ name, id }];
+});
+const storageItems = async (context, origin, isLocalStorage, limitations) => {
+    const raw = await optionalCdpCommand(context, "DOMStorage.getDOMStorageItems", { storageId: { securityOrigin: origin, isLocalStorage } }, limitations);
+    const entries = recordValue(raw)?.entries;
+    if (raw === undefined || !Array.isArray(entries))
+        return { items: [], complete: false };
+    const items = entries.flatMap((entry) => {
+        if (!Array.isArray(entry))
+            return [];
+        const key = cdpStringValue(entry[0]);
+        const value = cdpStringValue(entry[1]);
+        return key === undefined || value === undefined ? [] : [{ key, value }];
+    });
+    return { items, complete: items.length === entries.length };
+};
+const stringArray = (value) => Array.isArray(value)
+    ? value.flatMap((item) => {
+        const text = cdpStringValue(item);
+        return text === undefined ? [] : [text];
+    })
+    : [];

package/dist/browser/CdpCaptureStorageFingerprints.js

@@ -0,0 +1,320 @@
+import { createHash } from "node:crypto";
+import canonicalize from "canonicalize";
+import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
+import { CdpConnection } from "./CdpConnection.js";
+import { optionalCdpCommand } from "./CdpOptionalCommand.js";
+import { recordValue, recordsValue, cdpStringValue, } from "./CdpCaptureValues.js";
+/** Capture stable hashes of approved storage content without returning values. */
+export const captureStorageFingerprints = async ({ context, origin, cookieUrl, local, session, indexedDbNames, caches, limitations, }) => {
+    const state = {
+        items: [],
+        complete: local.complete && session.complete,
+    };
+    const run = { context, limitations, state };
+    addStorageItems(state, "local_storage", local.items);
+    addStorageItems(state, "session_storage", session.items);
+    await addCookies(run, cookieUrl);
+    await addIndexedDb(run, origin, indexedDbNames);
+    await addCaches(run, caches);
+    state.items.sort((left, right) => compareUnicodeCodePoints(`${left.scope}:${left.identity_sha256}`, `${right.scope}:${right.identity_sha256}`));
+    const complete = state.complete && state.items.every(({ complete }) => complete);
+    return { items: state.items, complete };
+};
+const addStorageItems = (state, scope, items) => {
+    for (const item of items)
+        addFingerprint(state, {
+            scope,
+            identity: item.key,
+            value: item.value,
+            complete: true,
+        });
+};
+const addCookies = async (run, pageUrl) => {
+    run.limitations.push("Cookie fingerprints cover cookies applicable to the current main-frame URL, not every path on its origin.");
+    const raw = await optionalCdpCommand(run.context, "Network.getCookies", { urls: [pageUrl] }, run.limitations);
+    if (raw === undefined) {
+        run.state.complete = false;
+        return;
+    }
+    const cookieValues = recordValue(raw)?.cookies;
+    if (!Array.isArray(cookieValues)) {
+        run.state.complete = false;
+        return;
+    }
+    const cookies = recordsValue(cookieValues);
+    if (cookies.length !== cookieValues.length)
+        run.state.complete = false;
+    for (const cookie of cookies) {
+        const name = cdpStringValue(cookie.name);
+        const value = cdpStringValue(cookie.value);
+        const domain = cdpStringValue(cookie.domain);
+        const path = cdpStringValue(cookie.path);
+        if (name === undefined ||
+            value === undefined ||
+            domain === undefined ||
+            path === undefined) {
+            run.state.complete = false;
+            continue;
+        }
+        addFingerprint(run.state, {
+            scope: "cookie",
+            identity: {
+                domain,
+                path,
+                name,
+                partition_key: cookie.partitionKey ?? null,
+            },
+            value,
+            complete: true,
+        });
+    }
+};
+const addIndexedDb = async (run, origin, names) => {
+    for (const databaseName of names) {
+        const raw = await optionalCdpCommand(run.context, "IndexedDB.requestDatabase", { securityOrigin: origin, databaseName }, run.limitations);
+        const database = recordValue(recordValue(raw)?.databaseWithObjectStores);
+        if (raw === undefined || database === undefined) {
+            run.state.complete = false;
+            continue;
+        }
+        if (cdpStringValue(database.name) !== databaseName) {
+            run.state.complete = false;
+            continue;
+        }
+        if (!Array.isArray(database.objectStores)) {
+            run.state.complete = false;
+            continue;
+        }
+        const stores = recordsValue(database.objectStores);
+        if (stores.length !== database.objectStores.length)
+            run.state.complete = false;
+        addFingerprint(run.state, {
+            scope: "indexed_db_schema",
+            identity: databaseName,
+            value: stores.map(indexedDbStoreSchema),
+            complete: indexedDbSchemaComplete(stores),
+        });
+        for (const store of stores) {
+            const objectStoreName = cdpStringValue(store.name);
+            if (objectStoreName === undefined) {
+                run.state.complete = false;
+                continue;
+            }
+            await addIndexedDbRecords(run, {
+                origin,
+                databaseName,
+                objectStoreName,
+            });
+        }
+    }
+};
+const indexedDbStoreSchema = (store) => ({
+    name: cdpStringValue(store.name) ?? null,
+    key_path: stableValue(store.keyPath),
+    auto_increment: store.autoIncrement === true,
+    indexes: recordsValue(store.indexes).map((index) => ({
+        name: cdpStringValue(index.name) ?? null,
+        key_path: stableValue(index.keyPath),
+        unique: index.unique === true,
+        multi_entry: index.multiEntry === true,
+    })),
+});
+const indexedDbSchemaComplete = (stores) => stores.every((store) => cdpStringValue(store.name) !== undefined &&
+    stableValueComplete(store.keyPath) &&
+    typeof store.autoIncrement === "boolean" &&
+    Array.isArray(store.indexes) &&
+    recordsValue(store.indexes).length === store.indexes.length &&
+    recordsValue(store.indexes).every((index) => cdpStringValue(index.name) !== undefined &&
+        stableValueComplete(index.keyPath) &&
+        typeof index.unique === "boolean" &&
+        typeof index.multiEntry === "boolean"));
+const addIndexedDbRecords = async (run, input) => {
+    let skipCount = 0;
+    while (true) {
+        const raw = await optionalCdpCommand(run.context, "IndexedDB.requestData", {
+            securityOrigin: input.origin,
+            databaseName: input.databaseName,
+            objectStoreName: input.objectStoreName,
+            skipCount,
+            pageSize: STORAGE_ENTRY_PAGE_SIZE,
+        }, run.limitations);
+        const result = recordValue(raw);
+        if (raw === undefined || result === undefined) {
+            run.state.complete = false;
+            return;
+        }
+        if (!Array.isArray(result.objectStoreDataEntries) ||
+            typeof result.hasMore !== "boolean") {
+            run.state.complete = false;
+            return;
+        }
+        const entries = recordsValue(result.objectStoreDataEntries);
+        if (entries.length !== result.objectStoreDataEntries.length)
+            run.state.complete = false;
+        for (const entry of entries) {
+            const key = stableValue(entry.key);
+            const primaryKey = stableValue(entry.primaryKey);
+            const value = stableValue(entry.value);
+            addFingerprint(run.state, {
+                scope: "indexed_db_record",
+                identity: {
+                    database: input.databaseName,
+                    store: input.objectStoreName,
+                    key,
+                    primaryKey,
+                },
+                value,
+                complete: remoteObjectComplete(entry.key) &&
+                    remoteObjectComplete(entry.primaryKey) &&
+                    remoteObjectComplete(entry.value),
+            });
+        }
+        if (result.hasMore !== true)
+            return;
+        if (entries.length === 0 ||
+            skipCount > Number.MAX_SAFE_INTEGER - entries.length) {
+            run.state.complete = false;
+            return;
+        }
+        skipCount += entries.length;
+    }
+};
+const addCaches = async (run, caches) => {
+    for (const cache of caches) {
+        let skipCount = 0;
+        let returnCount;
+        while (returnCount === undefined || skipCount < returnCount) {
+            const raw = await optionalCdpCommand(run.context, "CacheStorage.requestEntries", { cacheId: cache.id, skipCount, pageSize: STORAGE_ENTRY_PAGE_SIZE }, run.limitations);
+            const result = recordValue(raw);
+            if (raw === undefined || result === undefined) {
+                run.state.complete = false;
+                break;
+            }
+            if (!Array.isArray(result.cacheDataEntries) ||
+                typeof result.returnCount !== "number" ||
+                !Number.isSafeInteger(result.returnCount) ||
+                result.returnCount < 0) {
+                run.state.complete = false;
+                break;
+            }
+            const entries = recordsValue(result.cacheDataEntries);
+            if (entries.length !== result.cacheDataEntries.length)
+                run.state.complete = false;
+            for (const entry of entries)
+                await addCacheEntry(run, cache, entry);
+            returnCount = result.returnCount;
+            if (skipCount + entries.length >= returnCount)
+                break;
+            if (entries.length === 0 ||
+                skipCount > Number.MAX_SAFE_INTEGER - entries.length) {
+                run.state.complete = false;
+                break;
+            }
+            skipCount += entries.length;
+        }
+    }
+};
+const addCacheEntry = async (run, cache, entry) => {
+    const requestURL = cdpStringValue(entry.requestURL);
+    const requestMethod = cdpStringValue(entry.requestMethod);
+    if (requestURL === undefined || requestMethod === undefined) {
+        run.state.complete = false;
+        return;
+    }
+    if (!Array.isArray(entry.requestHeaders)) {
+        run.state.complete = false;
+        return;
+    }
+    const headerRecords = recordsValue(entry.requestHeaders);
+    const requestHeaders = headerRecords.flatMap((header) => {
+        const name = cdpStringValue(header.name);
+        const value = cdpStringValue(header.value);
+        return name === undefined || value === undefined ? [] : [{ name, value }];
+    });
+    if (headerRecords.length !== entry.requestHeaders.length ||
+        requestHeaders.length !== headerRecords.length) {
+        run.state.complete = false;
+        return;
+    }
+    const raw = await optionalCdpCommand(run.context, "CacheStorage.requestCachedResponse", { cacheId: cache.id, requestURL, requestHeaders }, run.limitations);
+    const body = cdpStringValue(recordValue(recordValue(raw)?.response)?.body);
+    const decoded = body === undefined ? null : decodeBase64(body);
+    const complete = decoded !== null;
+    if (!complete)
+        run.state.complete = false;
+    addFingerprint(run.state, {
+        scope: "cache_entry",
+        identity: { cache: cache.name, requestURL, requestMethod, requestHeaders },
+        value: decoded ?? undefined,
+        complete,
+    });
+};
+const addFingerprint = (state, input) => {
+    state.items.push({
+        scope: input.scope,
+        identity_sha256: digest(input.identity),
+        value_sha256: input.value === undefined ? null : digest(input.value),
+        complete: input.complete &&
+            stableValueComplete(input.identity) &&
+            stableValueComplete(input.value),
+    });
+};
+const STORAGE_ENTRY_PAGE_SIZE = 1_000;
+const digest = (value) => createHash("sha256").update(encodedValue(value)).digest("hex");
+const encodedValue = (value) => {
+    if (value instanceof Uint8Array)
+        return value;
+    const encoded = canonicalize(stableValue(value));
+    if (encoded === undefined)
+        throw new TypeError("Expected canonical JSON");
+    return encoded;
+};
+const stableValue = (value) => {
+    if (value === null ||
+        typeof value === "string" ||
+        typeof value === "number" ||
+        typeof value === "boolean")
+        return value;
+    if (Array.isArray(value))
+        return value.map((item) => stableValue(item));
+    const record = recordValue(value);
+    if (record === undefined)
+        return String(value);
+    return Object.fromEntries(Object.entries(record)
+        .filter(([key]) => key !== "objectId" && key !== "customPreview")
+        .sort(([left], [right]) => compareUnicodeCodePoints(left, right))
+        .map(([key, item]) => [key, stableValue(item)]));
+};
+const stableValueComplete = (value) => {
+    if (value instanceof Uint8Array)
+        return true;
+    if (value === null ||
+        value === undefined ||
+        typeof value === "string" ||
+        typeof value === "number" ||
+        typeof value === "boolean")
+        return true;
+    if (Array.isArray(value))
+        return value.every((item) => stableValueComplete(item));
+    const record = recordValue(value);
+    if (record === undefined)
+        return false;
+    const entries = Object.entries(record).filter(([key]) => key !== "objectId" && key !== "customPreview");
+    return entries.every(([, item]) => stableValueComplete(item));
+};
+const remoteObjectComplete = (value) => {
+    const record = recordValue(value);
+    if (record === undefined)
+        return true;
+    return (record.objectId === undefined &&
+        record.preview === undefined &&
+        record.customPreview === undefined &&
+        (record.type !== "object" || record.value !== undefined) &&
+        stableValueComplete(record));
+};
+const decodeBase64 = (value) => {
+    if (value.length % 4 !== 0 ||
+        !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(value))
+        return null;
+    return Buffer.from(value, "base64");
+};