rea-agents 2.0.0 → 2.1.0

package/dist/browser/CdpCaptureEvents.js

@@ -1,12 +1,11 @@
-import { inferJsonShape } from "../domain/jsonShape.js";
 import { CdpCaptureCompleteness } from "./CdpCaptureCompleteness.js";
-import { safeResponseMetadata } from "./CdpSafeMetadata.js";
-import { boundedSensitiveText } from "./SensitiveTextCapture.js";
-import { allowedSanitizedUrl, boundedText, isHttpUrl, numberValue, recordValue, recordsValue, stringValue, } from "./CdpCaptureValues.js";
+import * as bodyShapes from "./CdpCaptureEventBodyShapes.js";
+import * as handlers from "./CdpCaptureEventHandlers.js";
+import { isJsonMediaType } from "./CdpCaptureEventHelpers.js";
+import { recordValue } from "./CdpCaptureValues.js";
+export {} from "./CdpCaptureEventTypes.js";
 /** Bounded event accumulator that drops payload values at ingestion time. */
 export class CdpCaptureEvents {
-    input;
-    allowedOrigins;
     scripts = new Map();
     executionContextFrames = new Map();
     network = new Map();
@@ -24,16 +23,18 @@ export class CdpCaptureEvents {
     ]);
     originViolation = false;
     navigationDuringCapture = false;
-    #mainFrameId;
-    #consoleTextBytes = 0;
-    #jsonBodyBytes = 0;
-    #websocketShapeBytes = 0;
+    mainFrameId = undefined;
+    consoleTextBytes = 0;
+    jsonBodyBytes = 0;
+    websocketShapeBytes = 0;
+    input;
+    allowedOrigins;
     constructor(input, allowedOrigins) {
         this.input = input;
         this.allowedOrigins = allowedOrigins;
     }
     beginAuthorizedFrame(frameId) {
-        this.#mainFrameId = frameId;
+        this.mainFrameId = frameId;
         this.originViolation = false;
         this.navigationDuringCapture = false;
         this.scripts.clear();
@@ -44,9 +45,9 @@ export class CdpCaptureEvents {
         this.websockets.length = 0;
         this.responseMetadata.length = 0;
         this.agentHints.length = 0;
-        this.#consoleTextBytes = 0;
-        this.#jsonBodyBytes = 0;
-        this.#websocketShapeBytes = 0;
+        this.consoleTextBytes = 0;
+        this.jsonBodyBytes = 0;
+        this.websocketShapeBytes = 0;
         this.completeness.reset();
         if (!this.input.include_console_text)
             this.completeness.exclude("console_text", "not_approved", null);
@@ -56,7 +57,7 @@ export class CdpCaptureEvents {
             this.completeness.exclude("websocket_shapes", "not_approved", null);
     }
     beginFinalCapture(frameId) {
-        this.#mainFrameId = frameId;
+        this.mainFrameId = frameId;
         this.navigationDuringCapture = false;
     }
     ingest(event) {
@@ -65,40 +66,40 @@ export class CdpCaptureEvents {
             return;
         switch (event.method) {
             case "Runtime.executionContextCreated":
-                this.#executionContext(params);
+                handlers.handleExecutionContextCreated(this, params);
                 break;
             case "Runtime.executionContextDestroyed":
-                this.#destroyExecutionContext(params);
+                handlers.handleExecutionContextDestroyed(this, params);
                 break;
             case "Runtime.executionContextsCleared":
                 this.executionContextFrames.clear();
                 break;
             case "Debugger.scriptParsed":
-                this.#script(params);
+                handlers.handleScriptParsed(this, params);
                 break;
             case "Network.requestWillBeSent":
-                this.#request(params);
+                handlers.handleRequestWillBeSent(this, params);
                 break;
             case "Network.responseReceived":
-                this.#response(params);
+                handlers.handleResponseReceived(this, params);
                 break;
             case "Network.loadingFinished":
-                this.#finished(params);
+                handlers.handleLoadingFinished(this, params);
                 break;
             case "Runtime.consoleAPICalled":
-                this.#console(params);
+                handlers.handleConsoleAPICalled(this, params);
                 break;
             case "Page.frameNavigated":
-                this.#frameNavigated(params);
+                handlers.handleFrameNavigated(this, params);
                 break;
             case "Network.webSocketCreated":
-                this.#websocketCreated(params);
+                handlers.handleWebSocketCreated(this, params);
                 break;
             case "Network.webSocketFrameSent":
-                this.#websocket(params, "sent");
+                handlers.handleWebSocketFrame(this, params, "sent");
                 break;
             case "Network.webSocketFrameReceived":
-                this.#websocket(params, "received");
+                handlers.handleWebSocketFrame(this, params, "received");
                 break;
         }
     }
@@ -111,220 +112,6 @@ export class CdpCaptureEvents {
             ? frameId
             : null;
     }
-    #executionContext(params) {
-        const context = recordValue(params.context);
-        const identifier = numberValue(context?.id);
-        const frameId = stringValue(recordValue(context?.auxData)?.frameId);
-        if (identifier === undefined ||
-            !Number.isSafeInteger(identifier) ||
-            frameId === undefined ||
-            frameId.length > 256)
-            return;
-        const key = String(identifier);
-        if (this.executionContextFrames.size >= this.input.limits.max_scripts &&
-            !this.executionContextFrames.has(key)) {
-            this.completeness.truncate("scripts");
-            return;
-        }
-        this.executionContextFrames.set(key, frameId);
-    }
-    #destroyExecutionContext(params) {
-        const key = executionContextKey(params.executionContextId);
-        if (key !== null)
-            this.executionContextFrames.delete(key);
-    }
-    #script(params) {
-        const scriptId = stringValue(params.scriptId);
-        const rawUrl = stringValue(params.url) ?? "";
-        const sanitized = allowedSanitizedUrl(rawUrl, this.allowedOrigins);
-        if (scriptId === undefined || scriptId.length > 256) {
-            this.completeness.exclude("scripts", "invalid_protocol_value");
-            return;
-        }
-        if (sanitized === undefined) {
-            this.completeness.exclude("scripts", exclusionReasonForUrl(rawUrl));
-            return;
-        }
-        if (this.scripts.size >= this.input.limits.max_scripts &&
-            !this.scripts.has(scriptId)) {
-            this.completeness.drop("scripts");
-            return;
-        }
-        const sourceMap = this.#sourceMap(params.sourceMapURL, rawUrl);
-        this.scripts.set(scriptId, {
-            scriptId,
-            rawUrl,
-            url: sanitized.url,
-            origin: sanitized.origin,
-            hash: (stringValue(params.hash) ?? "").slice(0, 512),
-            length: Math.max(0, Math.min(Number.MAX_SAFE_INTEGER, Math.trunc(numberValue(params.length) ?? 0))),
-            isModule: params.isModule === true,
-            language: boundedText(params.scriptLanguage, 100),
-            sourceMapUrl: sourceMap.sanitized,
-            sourceMapRawUrl: sourceMap.raw,
-            executionContextKey: executionContextKey(params.executionContextId),
-        });
-    }
-    #sourceMap(value, scriptUrl) {
-        const declaredUrl = stringValue(value);
-        if (declaredUrl === undefined || declaredUrl === "")
-            return { sanitized: null, raw: null };
-        let rawUrl;
-        try {
-            rawUrl = new URL(declaredUrl, scriptUrl).href;
-        }
-        catch {
-            this.completeness.exclude("source_maps", "unsupported_url");
-            return { sanitized: null, raw: null };
-        }
-        const sanitized = allowedSanitizedUrl(rawUrl, this.allowedOrigins);
-        if (sanitized !== undefined)
-            return { sanitized: sanitized.url, raw: rawUrl };
-        this.completeness.exclude("source_maps", exclusionReasonForUrl(rawUrl));
-        return { sanitized: null, raw: null };
-    }
-    #request(params) {
-        const requestId = stringValue(params.requestId);
-        const request = recordValue(params.request);
-        if (requestId === undefined || requestId.length > 256) {
-            this.completeness.exclude("network_requests", "invalid_protocol_value");
-            return;
-        }
-        if (request === undefined) {
-            this.completeness.exclude("network_requests", "invalid_protocol_value");
-            this.network.delete(requestId);
-            return;
-        }
-        const sanitized = allowedSanitizedUrl(request.url, this.allowedOrigins);
-        if (sanitized === undefined) {
-            this.completeness.exclude("network_requests", exclusionReasonForUrl(stringValue(request.url)));
-            this.network.delete(requestId);
-            return;
-        }
-        if (this.network.size >= this.input.limits.max_network_events &&
-            !this.network.has(requestId)) {
-            this.completeness.drop("network_requests");
-            return;
-        }
-        const initiator = recordValue(params.initiator);
-        const initiatorFrame = initiatorLocation(initiator);
-        const rawInitiatorUrl = stringValue(initiatorFrame?.url);
-        const initiatorUrl = allowedSanitizedUrl(rawInitiatorUrl, this.allowedOrigins);
-        if (rawInitiatorUrl !== undefined && initiatorUrl === undefined)
-            this.completeness.exclude("network_initiators", exclusionReasonForUrl(rawInitiatorUrl));
-        this.network.set(requestId, {
-            request_id: requestId,
-            url: sanitized.url,
-            origin: sanitized.origin ?? "",
-            method: (stringValue(request.method) ?? "GET").slice(0, 32),
-            resource_type: boundedText(params.type, 100),
-            status: null,
-            mime_type: null,
-            encoded_data_length: null,
-            initiator: {
-                type: (stringValue(initiator?.type) ?? "other").slice(0, 100),
-                url: initiatorUrl?.url ?? null,
-                line: integerOrNull(initiatorFrame?.lineNumber),
-                column: integerOrNull(initiatorFrame?.columnNumber),
-            },
-            body_shapes: this.#requestBodyShape(request),
-        });
-    }
-    #response(params) {
-        const requestId = stringValue(params.requestId);
-        if (requestId === undefined || requestId.length > 256) {
-            this.completeness.exclude("network_requests", "invalid_protocol_value");
-            return;
-        }
-        const current = this.network.get(requestId);
-        const response = recordValue(params.response);
-        if (current === undefined)
-            return;
-        const sanitized = allowedSanitizedUrl(response?.url, this.allowedOrigins);
-        if (response === undefined || sanitized === undefined) {
-            this.completeness.exclude("network_requests", response === undefined
-                ? "invalid_protocol_value"
-                : exclusionReasonForUrl(stringValue(response.url)));
-            this.network.delete(requestId);
-            return;
-        }
-        this.network.set(requestId, {
-            ...current,
-            status: numberValue(response.status) ?? null,
-            mime_type: boundedText(response.mimeType, 256),
-        });
-        if (this.responseMetadata.length >= this.input.limits.max_network_events) {
-            this.completeness.truncate("metadata");
-            return;
-        }
-        const metadata = safeResponseMetadata(requestId, sanitized.url, response, this.allowedOrigins);
-        this.responseMetadata.push(metadata.response);
-        this.agentHints.push(...metadata.agentHints);
-    }
-    #finished(params) {
-        const requestId = stringValue(params.requestId);
-        if (requestId === undefined || requestId.length > 256)
-            return;
-        const current = this.network.get(requestId);
-        if (current === undefined)
-            return;
-        this.network.set(requestId, {
-            ...current,
-            encoded_data_length: Math.max(0, numberValue(params.encodedDataLength) ?? 0),
-        });
-    }
-    #console(params) {
-        const frame = firstCallFrame(recordValue(params.stackTrace));
-        if (frame === undefined) {
-            this.completeness.exclude("console_events", "unattributed_origin");
-            return;
-        }
-        const source = allowedSanitizedUrl(frame.url, this.allowedOrigins);
-        if (source === undefined) {
-            this.completeness.exclude("console_events", exclusionReasonForUrl(stringValue(frame.url)));
-            return;
-        }
-        if (this.console.length >= this.input.limits.max_console_events) {
-            this.completeness.drop("console_events");
-            return;
-        }
-        const arguments_ = recordsValue(params.args);
-        this.console.push({
-            type: (stringValue(params.type) ?? "unknown").slice(0, 100),
-            timestamp: numberValue(params.timestamp) ?? 0,
-            argument_types: arguments_
-                .map((argument) => (stringValue(argument.type) ?? "unknown").slice(0, 100))
-                .slice(0, 100),
-            url: source.url,
-            line: integerOrNull(frame.lineNumber),
-            column: integerOrNull(frame.columnNumber),
-            text_capture: this.#consoleText(arguments_),
-        });
-    }
-    #websocket(params, direction) {
-        const requestId = stringValue(params.requestId);
-        if (requestId === undefined ||
-            requestId.length > 256 ||
-            (!this.network.has(requestId) && !this.allowedWebSockets.has(requestId)))
-            return;
-        if (this.websockets.length >= this.input.limits.max_websocket_events) {
-            this.completeness.drop("websocket_frames");
-            return;
-        }
-        const response = recordValue(params.response);
-        const payload = stringValue(response?.payloadData) ?? "";
-        const opcode = Math.max(0, Math.trunc(numberValue(response?.opcode) ?? 0));
-        const decoded = opcode === 1 ? undefined : decodeBase64(payload);
-        if (opcode !== 1 && decoded === undefined)
-            this.completeness.exclude("websocket_frames", "invalid_protocol_value");
-        this.websockets.push({
-            request_id: requestId,
-            direction,
-            opcode,
-            payload_bytes: opcode === 1 ? Buffer.byteLength(payload) : (decoded?.byteLength ?? 0),
-            payload_shape: this.#websocketShape(payload, opcode),
-        });
-    }
     responseBodyRequestIds() {
         if (!this.input.include_json_body_shapes)
             return [];
@@ -333,264 +120,10 @@ export class CdpCaptureEvents {
             .map((request) => request.request_id);
     }
     responseBodyUnavailable(requestId) {
-        this.#updateResponseShape(requestId, null, false);
+        bodyShapes.updateResponseBodyShape(this, requestId, null, false);
         this.completeness.unavailable("json_body_shapes");
     }
     ingestResponseBody(requestId, value) {
-        const result = recordValue(value);
-        const body = stringValue(result?.body);
-        if (body === undefined) {
-            this.#invalidResponseBody(requestId);
-            return;
-        }
-        const decoded = result?.base64Encoded === true
-            ? decodeBase64(body)?.toString("utf8")
-            : body;
-        if (decoded === undefined) {
-            this.#invalidResponseBody(requestId);
-            return;
-        }
-        const inferred = this.#inferBodyShape(decoded);
-        this.#updateResponseShape(requestId, inferred.shape, inferred.truncated);
-    }
-    #invalidResponseBody(requestId) {
-        this.#updateResponseShape(requestId, null, false);
-        this.completeness.exclude("json_body_shapes", "invalid_protocol_value");
-    }
-    #requestBodyShape(request) {
-        if (!this.input.include_json_body_shapes)
-            return { status: "not_approved", request: null, response: null };
-        const body = stringValue(request.postData);
-        if (!isJsonContentType(recordValue(request.headers)) || body === undefined)
-            return { status: "unavailable", request: null, response: null };
-        const inferred = this.#inferBodyShape(body);
-        return {
-            status: inferred.truncated
-                ? "truncated"
-                : inferred.shape === null
-                    ? "unavailable"
-                    : "included",
-            request: inferred.shape,
-            response: null,
-        };
-    }
-    #inferBodyShape(text) {
-        const bytes = Buffer.byteLength(text);
-        const remaining = this.input.limits.max_total_json_body_bytes - this.#jsonBodyBytes;
-        if (bytes > this.input.limits.max_json_body_bytes || bytes > remaining) {
-            this.completeness.truncate("json_body_shapes");
-            return { shape: null, truncated: true };
-        }
-        this.#jsonBodyBytes += bytes;
-        const shape = inferJsonShape(text, {
-            maximumBytes: this.input.limits.max_json_body_bytes,
-            maximumNodes: this.input.limits.max_json_shape_nodes,
-            maximumDepth: this.input.limits.max_json_shape_depth,
-        });
-        if (shape?.truncated === true)
-            this.completeness.truncate("json_body_shapes");
-        return { shape, truncated: shape?.truncated === true };
-    }
-    #updateResponseShape(requestId, response, truncated) {
-        const current = this.network.get(requestId);
-        if (current === undefined)
-            return;
-        const request = current.body_shapes.request;
-        const status = truncated || current.body_shapes.status === "truncated"
-            ? "truncated"
-            : response !== null
-                ? "included"
-                : request !== null
-                    ? "partial"
-                    : "unavailable";
-        this.network.set(requestId, {
-            ...current,
-            body_shapes: { status, request, response },
-        });
-    }
-    #consoleText(arguments_) {
-        if (!this.input.include_console_text)
-            return {
-                status: "not_approved",
-                values: [],
-                retained_bytes: 0,
-                truncated_values: 0,
-            };
-        const values = [];
-        let retainedBytes = 0;
-        let truncatedValues = 0;
-        for (const [argumentIndex, argument] of arguments_.entries()) {
-            const primitive = consolePrimitive(argument);
-            if (primitive === undefined)
-                continue;
-            const remaining = this.input.limits.max_total_console_text_bytes - this.#consoleTextBytes;
-            if (remaining <= 0) {
-                truncatedValues += 1;
-                continue;
-            }
-            const bounded = boundedSensitiveText(primitive.text, Math.min(this.input.limits.max_console_text_field_bytes, remaining));
-            values.push({
-                argument_index: argumentIndex,
-                type: primitive.type,
-                text: bounded.text,
-            });
-            retainedBytes += bounded.bytes;
-            this.#consoleTextBytes += bounded.bytes;
-            if (bounded.truncated)
-                truncatedValues += 1;
-        }
-        if (truncatedValues > 0)
-            this.completeness.truncate("console_text");
-        return {
-            status: truncatedValues > 0 ? "truncated" : "included",
-            values,
-            retained_bytes: retainedBytes,
-            truncated_values: truncatedValues,
-        };
-    }
-    #websocketShape(payload, opcode) {
-        if (!this.input.include_websocket_shapes)
-            return null;
-        if (opcode !== 1)
-            return { format: "binary", json_shape: null, truncated: false };
-        const bytes = Buffer.byteLength(payload);
-        const remaining = this.input.limits.max_total_websocket_shape_bytes -
-            this.#websocketShapeBytes;
-        if (bytes > this.input.limits.max_websocket_shape_bytes ||
-            bytes > remaining) {
-            this.completeness.truncate("websocket_shapes");
-            return { format: "text", json_shape: null, truncated: true };
-        }
-        this.#websocketShapeBytes += bytes;
-        const shape = inferJsonShape(payload, {
-            maximumBytes: this.input.limits.max_websocket_shape_bytes,
-            maximumNodes: this.input.limits.max_json_shape_nodes,
-            maximumDepth: this.input.limits.max_json_shape_depth,
-        });
-        if (shape?.truncated === true)
-            this.completeness.truncate("websocket_shapes");
-        return {
-            format: shape === null ? "text" : "json",
-            json_shape: shape,
-            truncated: shape?.truncated === true,
-        };
-    }
-    #websocketCreated(params) {
-        const requestId = stringValue(params.requestId);
-        const rawUrl = stringValue(params.url);
-        if (requestId === undefined || requestId.length > 256) {
-            this.completeness.exclude("websocket_connections", "invalid_protocol_value");
-            return;
-        }
-        if (rawUrl === undefined) {
-            this.completeness.exclude("websocket_connections", "invalid_protocol_value");
-            return;
-        }
-        let parsed;
-        try {
-            parsed = new URL(rawUrl);
-        }
-        catch {
-            this.completeness.exclude("websocket_connections", "unsupported_url");
-            return;
-        }
-        if (parsed.protocol === "ws:")
-            parsed.protocol = "http:";
-        else if (parsed.protocol === "wss:")
-            parsed.protocol = "https:";
-        else {
-            this.completeness.exclude("websocket_connections", "unsupported_url");
-            return;
-        }
-        if (!this.allowedOrigins.has(parsed.origin)) {
-            this.completeness.exclude("websocket_connections", "disallowed_origin");
-            return;
-        }
-        if (!this.allowedWebSockets.has(requestId) &&
-            this.allowedWebSockets.size >= this.input.limits.max_websocket_events) {
-            this.completeness.drop("websocket_connections");
-            return;
-        }
-        this.allowedWebSockets.add(requestId);
-    }
-    #frameNavigated(params) {
-        const frame = recordValue(params.frame);
-        if (stringValue(frame?.id) !== this.#mainFrameId)
-            return;
-        this.navigationDuringCapture = true;
-        const rawUrl = stringValue(frame?.url);
-        if (isHttpUrl(rawUrl) &&
-            allowedSanitizedUrl(rawUrl, this.allowedOrigins) === undefined)
-            this.originViolation = true;
+        bodyShapes.ingestResponseBodyShape(this, requestId, value);
     }
 }
-const firstCallFrame = (value) => recordsValue(value?.callFrames)[0];
-const initiatorLocation = (initiator) => {
-    let stack = recordValue(initiator?.stack);
-    for (let depth = 0; depth < 32 && stack !== undefined; depth += 1) {
-        const frame = firstCallFrame(stack);
-        if (frame !== undefined)
-            return frame;
-        stack = recordValue(stack.parent);
-    }
-    return stringValue(initiator?.url) === undefined ? undefined : initiator;
-};
-const exclusionReasonForUrl = (value) => value === undefined || value === ""
-    ? "unattributed_origin"
-    : isHttpUrl(value)
-        ? "disallowed_origin"
-        : "unsupported_url";
-const integerOrNull = (value) => {
-    const number = numberValue(value);
-    return number === undefined ? null : Math.max(0, Math.trunc(number));
-};
-const executionContextKey = (value) => {
-    const identifier = numberValue(value);
-    return identifier !== undefined && Number.isSafeInteger(identifier)
-        ? String(identifier)
-        : null;
-};
-const isJsonContentType = (headers) => {
-    if (headers === undefined)
-        return false;
-    for (const [name, value] of Object.entries(headers))
-        if (name.toLowerCase() === "content-type" &&
-            isJsonMediaType(stringValue(value)))
-            return true;
-    return false;
-};
-const isJsonMediaType = (value) => {
-    const mediaType = value?.split(";", 1)[0]?.trim().toLowerCase();
-    return (mediaType === "application/json" || mediaType?.endsWith("+json") === true);
-};
-const consolePrimitive = (value) => {
-    const type = stringValue(value.type);
-    switch (type) {
-        case "string":
-            return typeof value.value === "string"
-                ? { type, text: value.value }
-                : undefined;
-        case "boolean":
-            return typeof value.value === "boolean"
-                ? { type, text: String(value.value) }
-                : undefined;
-        case "number":
-            return typeof value.value === "number"
-                ? { type, text: String(value.value) }
-                : typeof value.unserializableValue === "string"
-                    ? { type, text: value.unserializableValue }
-                    : undefined;
-        case "bigint":
-            return typeof value.unserializableValue === "string"
-                ? { type, text: value.unserializableValue }
-                : undefined;
-        case "undefined":
-            return { type, text: "undefined" };
-        default:
-            return undefined;
-    }
-};
-const BASE64 = /^(?:[A-Za-z\d+/]{4})*(?:[A-Za-z\d+/]{2}==|[A-Za-z\d+/]{3}=)?$/u;
-const decodeBase64 = (value) => value.length % 4 === 0 && BASE64.test(value)
-    ? Buffer.from(value, "base64")
-    : undefined;