rea-agents 1.3.0 → 1.5.0

package/dist/browser/CdpCaptureDocuments.js

@@ -0,0 +1,140 @@
+import { allowedSanitizedUrl, boundedText, numberValue, recordValue, recordsValue, requiredRecord, stringValue, } from "./CdpCaptureValues.js";
+/** Normalize allowed frames while retaining a bounded prefix and total count. */
+export const captureFrames = (result, allowedOrigins, maximum) => {
+    const root = recordValue(requiredRecord(result).frameTree);
+    if (root === undefined)
+        return { total: 0, items: [] };
+    const items = [];
+    let total = 0;
+    for (const tree of walkFrameTrees(root)) {
+        const frame = recordValue(tree.frame);
+        const frameId = stringValue(frame?.id);
+        const sanitized = allowedSanitizedUrl(frame?.url, allowedOrigins);
+        if (frame === undefined ||
+            frameId === undefined ||
+            frameId.length > 256 ||
+            sanitized === undefined ||
+            sanitized.origin === null)
+            continue;
+        total += 1;
+        if (items.length >= maximum)
+            continue;
+        items.push({
+            frame_id: frameId,
+            parent_frame_id: (stringValue(frame.parentId) ?? "").slice(0, 256) || null,
+            url: sanitized.url,
+            origin: sanitized.origin,
+        });
+    }
+    return { total, items };
+};
+/** Read the current main-frame URL from an untrusted Page.getFrameTree result. */
+export const mainFrameUrl = (result) => stringValue(recordValue(recordValue(requiredRecord(result).frameTree)?.frame)?.url);
+/** Normalize allowed resources while retaining a bounded prefix and total count. */
+export const captureResources = (result, allowedOrigins, maximum) => {
+    const root = recordValue(requiredRecord(result).frameTree);
+    if (root === undefined)
+        return { total: 0, items: [] };
+    const items = [];
+    let total = 0;
+    for (const tree of walkFrameTrees(root)) {
+        for (const resource of recordsValue(tree.resources)) {
+            const url = allowedSanitizedUrl(resource.url, allowedOrigins);
+            if (url === undefined || url.origin === null)
+                continue;
+            total += 1;
+            if (items.length >= maximum)
+                continue;
+            const contentSize = numberValue(resource.contentSize);
+            items.push({
+                url: url.url,
+                origin: url.origin,
+                type: (stringValue(resource.type) ?? "Other").slice(0, 100),
+                mime_type: (stringValue(resource.mimeType) ?? "").slice(0, 256),
+                content_size: contentSize === undefined ? null : Math.max(0, contentSize),
+            });
+        }
+    }
+    return { total, items };
+};
+const walkFrameTrees = function* (root) {
+    const pending = [root];
+    while (pending.length > 0) {
+        const tree = pending.pop();
+        if (tree === undefined)
+            return;
+        const children = recordsValue(tree.childFrames);
+        for (let index = children.length - 1; index >= 0; index -= 1) {
+            const child = children[index];
+            if (child !== undefined)
+                pending.push(child);
+        }
+        yield tree;
+    }
+};
+export const captureDom = (result, allowedOrigins, input) => {
+    const root = requiredRecord(result);
+    const strings = Array.isArray(root.strings)
+        ? root.strings.map((value) => stringValue(value) ?? "")
+        : [];
+    const nodes = [];
+    let total = 0;
+    for (const document of recordsValue(root.documents)) {
+        const documentUrl = indexedString(strings, document.documentURL);
+        if (allowedSanitizedUrl(documentUrl, allowedOrigins) === undefined)
+            continue;
+        const documentNodes = recordValue(document.nodes);
+        if (documentNodes === undefined)
+            continue;
+        const nodeTypes = numberArray(documentNodes.nodeType);
+        const nodeNames = numberArray(documentNodes.nodeName);
+        const nodeValues = numberArray(documentNodes.nodeValue);
+        const parents = numberArray(documentNodes.parentIndex);
+        const attributes = Array.isArray(documentNodes.attributes)
+            ? documentNodes.attributes
+            : [];
+        const baseIndex = nodes.length;
+        total += nodeTypes.length;
+        for (let index = 0; index < nodeTypes.length && nodes.length < input.limits.max_dom_nodes; index += 1) {
+            const attributeIndexes = numberArray(attributes[index]);
+            const parent = Math.trunc(parents[index] ?? -1);
+            nodes.push({
+                index: nodes.length,
+                parent_index: parent < 0 ? -1 : baseIndex + parent,
+                node_type: Math.max(0, Math.trunc(nodeTypes[index] ?? 0)),
+                node_name: indexedString(strings, nodeNames[index]).slice(0, 256),
+                node_value_length: indexedString(strings, nodeValues[index]).length,
+                attribute_names: attributeIndexes
+                    .filter((_value, attributeIndex) => attributeIndex % 2 === 0)
+                    .map((value) => indexedString(strings, value).slice(0, 256))
+                    .slice(0, 200),
+            });
+        }
+    }
+    return { total, nodes };
+};
+export const captureAccessibility = (results, maximum) => {
+    const all = results.flatMap((result) => recordsValue(requiredRecord(result).nodes));
+    return {
+        total: all.length,
+        nodes: all.slice(0, maximum).map((node) => ({
+            node_id: (stringValue(node.nodeId) ?? "").slice(0, 256),
+            parent_id: (stringValue(node.parentId) ?? "").slice(0, 256) || null,
+            role: axText(node.role),
+            name: axText(node.name),
+            description: axText(node.description),
+            ignored: node.ignored === true,
+        })),
+    };
+};
+const axText = (value) => boundedText(recordValue(value)?.value, 1_024);
+const indexedString = (strings, index) => {
+    const integer = numberValue(index);
+    return integer === undefined ? "" : (strings[Math.trunc(integer)] ?? "");
+};
+const numberArray = (value) => Array.isArray(value)
+    ? value.flatMap((item) => {
+        const number = numberValue(item);
+        return number === undefined ? [] : [number];
+    })
+    : [];

package/dist/browser/CdpCaptureEvents.js

@@ -0,0 +1,248 @@
+import { allowedSanitizedUrl, boundedText, isHttpUrl, numberValue, recordValue, recordsValue, stringValue, } from "./CdpCaptureValues.js";
+/** Bounded event accumulator that drops payload values at ingestion time. */
+export class CdpCaptureEvents {
+    input;
+    allowedOrigins;
+    scripts = new Map();
+    network = new Map();
+    allowedWebSockets = new Set();
+    console = [];
+    websockets = [];
+    dropped = 0;
+    originViolation = false;
+    navigationDuringCapture = false;
+    #mainFrameId;
+    constructor(input, allowedOrigins) {
+        this.input = input;
+        this.allowedOrigins = allowedOrigins;
+    }
+    beginAuthorizedFrame(frameId) {
+        this.#mainFrameId = frameId;
+        this.originViolation = false;
+        this.navigationDuringCapture = false;
+        this.dropped = 0;
+        this.scripts.clear();
+        this.network.clear();
+        this.allowedWebSockets.clear();
+        this.console.length = 0;
+        this.websockets.length = 0;
+    }
+    beginFinalCapture(frameId) {
+        this.#mainFrameId = frameId;
+        this.navigationDuringCapture = false;
+    }
+    ingest(event) {
+        const params = recordValue(event.params);
+        if (params === undefined)
+            return;
+        switch (event.method) {
+            case "Debugger.scriptParsed":
+                this.#script(params);
+                break;
+            case "Network.requestWillBeSent":
+                this.#request(params);
+                break;
+            case "Network.responseReceived":
+                this.#response(params);
+                break;
+            case "Network.loadingFinished":
+                this.#finished(params);
+                break;
+            case "Runtime.consoleAPICalled":
+                this.#console(params);
+                break;
+            case "Page.frameNavigated":
+                this.#frameNavigated(params);
+                break;
+            case "Network.webSocketCreated":
+                this.#websocketCreated(params);
+                break;
+            case "Network.webSocketFrameSent":
+                this.#websocket(params, "sent");
+                break;
+            case "Network.webSocketFrameReceived":
+                this.#websocket(params, "received");
+                break;
+        }
+    }
+    #script(params) {
+        const scriptId = stringValue(params.scriptId);
+        const rawUrl = stringValue(params.url) ?? "";
+        const sanitized = allowedSanitizedUrl(rawUrl, this.allowedOrigins);
+        if (scriptId === undefined ||
+            scriptId.length > 256 ||
+            sanitized === undefined)
+            return;
+        if (this.scripts.size >= this.input.limits.max_scripts &&
+            !this.scripts.has(scriptId)) {
+            this.dropped += 1;
+            return;
+        }
+        this.scripts.set(scriptId, {
+            scriptId,
+            url: sanitized.url,
+            origin: sanitized.origin,
+            hash: (stringValue(params.hash) ?? "").slice(0, 512),
+            length: Math.max(0, Math.min(Number.MAX_SAFE_INTEGER, Math.trunc(numberValue(params.length) ?? 0))),
+            isModule: params.isModule === true,
+            language: boundedText(params.scriptLanguage, 100),
+            sourceMapUrl: this.#sourceMap(params.sourceMapURL),
+        });
+    }
+    #sourceMap(value) {
+        const sanitized = allowedSanitizedUrl(value, this.allowedOrigins);
+        return sanitized?.url ?? null;
+    }
+    #request(params) {
+        const requestId = stringValue(params.requestId);
+        const request = recordValue(params.request);
+        const sanitized = allowedSanitizedUrl(request?.url, this.allowedOrigins);
+        if (requestId === undefined || requestId.length > 256)
+            return;
+        if (request === undefined || sanitized === undefined) {
+            this.network.delete(requestId);
+            return;
+        }
+        if (this.network.size >= this.input.limits.max_network_events &&
+            !this.network.has(requestId)) {
+            this.dropped += 1;
+            return;
+        }
+        const initiator = recordValue(params.initiator);
+        const initiatorUrl = allowedSanitizedUrl(firstCallFrame(initiator)?.url, this.allowedOrigins);
+        this.network.set(requestId, {
+            request_id: requestId,
+            url: sanitized.url,
+            origin: sanitized.origin ?? "",
+            method: (stringValue(request.method) ?? "GET").slice(0, 32),
+            resource_type: boundedText(params.type, 100),
+            status: null,
+            mime_type: null,
+            encoded_data_length: null,
+            initiator: {
+                type: (stringValue(initiator?.type) ?? "other").slice(0, 100),
+                url: initiatorUrl?.url ?? null,
+                line: integerOrNull(firstCallFrame(initiator)?.lineNumber),
+                column: integerOrNull(firstCallFrame(initiator)?.columnNumber),
+            },
+        });
+    }
+    #response(params) {
+        const requestId = stringValue(params.requestId);
+        if (requestId === undefined || requestId.length > 256)
+            return;
+        const current = this.network.get(requestId);
+        const response = recordValue(params.response);
+        const sanitized = allowedSanitizedUrl(response?.url, this.allowedOrigins);
+        if (current === undefined)
+            return;
+        if (response === undefined || sanitized === undefined) {
+            this.network.delete(requestId);
+            return;
+        }
+        this.network.set(requestId, {
+            ...current,
+            status: numberValue(response.status) ?? null,
+            mime_type: boundedText(response.mimeType, 256),
+        });
+    }
+    #finished(params) {
+        const requestId = stringValue(params.requestId);
+        if (requestId === undefined || requestId.length > 256)
+            return;
+        const current = this.network.get(requestId);
+        if (current === undefined)
+            return;
+        this.network.set(requestId, {
+            ...current,
+            encoded_data_length: Math.max(0, numberValue(params.encodedDataLength) ?? 0),
+        });
+    }
+    #console(params) {
+        const frame = firstCallFrame(recordValue(params.stackTrace));
+        const source = allowedSanitizedUrl(frame?.url, this.allowedOrigins);
+        if (frame === undefined || source === undefined)
+            return;
+        if (this.console.length >= this.input.limits.max_console_events) {
+            this.dropped += 1;
+            return;
+        }
+        this.console.push({
+            type: (stringValue(params.type) ?? "unknown").slice(0, 100),
+            timestamp: numberValue(params.timestamp) ?? 0,
+            argument_types: recordsValue(params.args)
+                .map((argument) => (stringValue(argument.type) ?? "unknown").slice(0, 100))
+                .slice(0, 100),
+            url: source?.url ?? null,
+            line: integerOrNull(frame?.lineNumber),
+            column: integerOrNull(frame?.columnNumber),
+        });
+    }
+    #websocket(params, direction) {
+        const requestId = stringValue(params.requestId);
+        if (requestId === undefined ||
+            requestId.length > 256 ||
+            (!this.network.has(requestId) && !this.allowedWebSockets.has(requestId)))
+            return;
+        if (this.websockets.length >= this.input.limits.max_websocket_events) {
+            this.dropped += 1;
+            return;
+        }
+        const response = recordValue(params.response);
+        const payload = stringValue(response?.payloadData) ?? "";
+        const opcode = Math.max(0, Math.trunc(numberValue(response?.opcode) ?? 0));
+        this.websockets.push({
+            request_id: requestId,
+            direction,
+            opcode,
+            payload_bytes: opcode === 1
+                ? Buffer.byteLength(payload)
+                : Buffer.from(payload, "base64").byteLength,
+        });
+    }
+    #websocketCreated(params) {
+        const requestId = stringValue(params.requestId);
+        const rawUrl = stringValue(params.url);
+        if (requestId === undefined ||
+            requestId.length > 256 ||
+            rawUrl === undefined)
+            return;
+        let parsed;
+        try {
+            parsed = new URL(rawUrl);
+        }
+        catch {
+            return;
+        }
+        if (parsed.protocol === "ws:")
+            parsed.protocol = "http:";
+        else if (parsed.protocol === "wss:")
+            parsed.protocol = "https:";
+        else
+            return;
+        if (this.allowedOrigins.has(parsed.origin) &&
+            (this.allowedWebSockets.has(requestId) ||
+                this.allowedWebSockets.size < this.input.limits.max_websocket_events))
+            this.allowedWebSockets.add(requestId);
+        else if (this.allowedOrigins.has(parsed.origin))
+            this.dropped += 1;
+    }
+    #frameNavigated(params) {
+        const frame = recordValue(params.frame);
+        if (stringValue(frame?.id) !== this.#mainFrameId)
+            return;
+        this.navigationDuringCapture = true;
+        const rawUrl = stringValue(frame?.url);
+        if (isHttpUrl(rawUrl) &&
+            allowedSanitizedUrl(rawUrl, this.allowedOrigins) === undefined)
+            this.originViolation = true;
+    }
+}
+const firstCallFrame = (value) => {
+    const callFrames = recordsValue(value?.callFrames);
+    return callFrames[0];
+};
+const integerOrNull = (value) => {
+    const number = numberValue(value);
+    return number === undefined ? null : Math.max(0, Math.trunc(number));
+};

package/dist/browser/CdpCaptureStorage.js

@@ -0,0 +1,53 @@
+import { CdpConnection } from "./CdpConnection.js";
+import { optionalCdpCommand } from "./CdpOptionalCommand.js";
+import { numberValue, recordValue, recordsValue, stringValue, } from "./CdpCaptureValues.js";
+/** Capture redacted, bounded storage metadata for one authorized origin. */
+export const captureStorage = async (context, origin, limitations) => {
+    const quota = recordValue(await optionalCdpCommand(context, "Storage.getUsageAndQuota", { origin }, limitations));
+    const local = context.input.include_storage_keys
+        ? await storageKeys(context, origin, true, limitations)
+        : [];
+    const session = context.input.include_storage_keys
+        ? await storageKeys(context, origin, false, limitations)
+        : [];
+    const indexed = context.input.include_storage_keys
+        ? stringArray(recordValue(await optionalCdpCommand(context, "IndexedDB.requestDatabaseNames", { securityOrigin: origin }, limitations))?.databaseNames)
+        : [];
+    const caches = context.input.include_storage_keys
+        ? recordsValue(recordValue(await optionalCdpCommand(context, "CacheStorage.requestCacheNames", { securityOrigin: origin }, limitations))?.caches).flatMap((cache) => {
+            const name = stringValue(cache.cacheName);
+            return name === undefined ? [] : [name.slice(0, 1_024)];
+        })
+        : [];
+    const maximum = context.input.limits.max_storage_keys;
+    return {
+        value: {
+            origin,
+            usage_bytes: numberValue(quota?.usage) ?? null,
+            quota_bytes: numberValue(quota?.quota) ?? null,
+            local_storage_keys: local.slice(0, maximum),
+            session_storage_keys: session.slice(0, maximum),
+            indexed_db_names: indexed.slice(0, maximum),
+            cache_names: caches.slice(0, maximum),
+            values_redacted: true,
+        },
+        truncated: [local, session, indexed, caches].some((items) => items.length > maximum),
+    };
+};
+const storageKeys = async (context, origin, isLocalStorage, limitations) => {
+    const result = recordValue(await optionalCdpCommand(context, "DOMStorage.getDOMStorageItems", { storageId: { securityOrigin: origin, isLocalStorage } }, limitations));
+    return Array.isArray(result?.entries)
+        ? result.entries.flatMap((entry) => {
+            if (!Array.isArray(entry))
+                return [];
+            const key = stringValue(entry[0]);
+            return key === undefined ? [] : [key.slice(0, 1_024)];
+        })
+        : [];
+};
+const stringArray = (value) => Array.isArray(value)
+    ? value.flatMap((item) => {
+        const text = stringValue(item);
+        return text === undefined ? [] : [text.slice(0, 1_024)];
+    })
+    : [];

package/dist/browser/CdpCaptureValues.js

@@ -0,0 +1,70 @@
+import { createHash } from "node:crypto";
+import { AnalysisCancelledError, BrowserObservationError, } from "../domain/errors.js";
+import { sanitizeBrowserUrl } from "../domain/browserObservation.js";
+const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+export const recordValue = (value) => isRecord(value) ? value : undefined;
+export const recordsValue = (value) => Array.isArray(value)
+    ? value.flatMap((item) => {
+        const record = recordValue(item);
+        return record === undefined ? [] : [record];
+    })
+    : [];
+export const stringValue = (value) => typeof value === "string" ? value : undefined;
+export const numberValue = (value) => typeof value === "number" && Number.isFinite(value) ? value : undefined;
+export const requiredRecord = (value) => {
+    const record = recordValue(value);
+    if (record === undefined)
+        throw new BrowserObservationError("inspect_web_page", "protocol_error");
+    return record;
+};
+export const boundedText = (value, maximum = 1_024) => {
+    const text = stringValue(value);
+    return text === undefined ? null : text.slice(0, maximum);
+};
+export const isHttpUrl = (value) => {
+    if (value === undefined)
+        return false;
+    try {
+        const protocol = new URL(value).protocol;
+        return protocol === "http:" || protocol === "https:";
+    }
+    catch {
+        return false;
+    }
+};
+export const allowedSanitizedUrl = (value, allowedOrigins) => {
+    const text = stringValue(value);
+    if (text === undefined)
+        return undefined;
+    const sanitized = sanitizeBrowserUrl(text);
+    return sanitized.origin !== null && allowedOrigins.has(sanitized.origin)
+        ? sanitized
+        : undefined;
+};
+export const sourceResult = (source) => ({
+    included: true,
+    sha256: createHash("sha256").update(source).digest("hex"),
+    bytes: Buffer.byteLength(source),
+    content: source,
+});
+export const sourceExcluded = (reason) => ({ included: false, reason });
+export const delayWithCancellation = async (durationMs, signal) => {
+    if (durationMs === 0)
+        return;
+    if (signal?.aborted === true)
+        throw new AnalysisCancelledError("inspect_web_page");
+    await new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            signal?.removeEventListener("abort", onAbort);
+            resolve();
+        }, durationMs);
+        const onAbort = () => {
+            clearTimeout(timer);
+            signal?.removeEventListener("abort", onAbort);
+            reject(new AnalysisCancelledError("inspect_web_page"));
+        };
+        signal?.addEventListener("abort", onAbort, { once: true });
+        if (signal?.aborted === true)
+            onAbort();
+    });
+};