rea-agents 1.4.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -13
- package/dist/application/AnalysisSnapshotCache.js +8 -6
- package/dist/application/ArtifactGraphConstruction.js +11 -2
- package/dist/application/ArtifactInventory.js +3 -2
- package/dist/application/AuthorizedArtifactInventory.js +2 -1
- package/dist/application/BinarySession.js +26 -11
- package/dist/application/BoundedJsonFiles.js +6 -29
- package/dist/application/BrowserEvidence.js +107 -0
- package/dist/application/BrowserObservationPort.js +1 -0
- package/dist/application/BrowserObservationService.js +116 -0
- package/dist/application/CapabilityInventory.js +14 -0
- package/dist/application/ClientConfigPath.js +33 -0
- package/dist/application/ConfiguredRoots.js +19 -0
- package/dist/application/CrossVersionInventory.js +16 -3
- package/dist/application/CrossVersionInvestigation.js +23 -62
- package/dist/application/CrossVersionInvestigationEvidence.js +64 -0
- package/dist/application/CrossVersionInvestigationReplay.js +170 -0
- package/dist/application/DeferredFileAuthorization.js +61 -0
- package/dist/application/DirectAnalysis.js +38 -18
- package/dist/application/Doctor.js +16 -11
- package/dist/application/ElectronEvidence.js +35 -0
- package/dist/application/ElectronObservationPort.js +1 -0
- package/dist/application/ElectronObservationService.js +46 -0
- package/dist/application/EvidenceLedger.js +11 -7
- package/dist/application/InvestigationWorkspaceStore.js +2 -26
- package/dist/application/PermissionAuthority.js +30 -0
- package/dist/application/ProcessCaptureAuthority.js +3 -2
- package/dist/application/ProcessCaptureLifecycle.js +19 -13
- package/dist/application/ProcessCheckpoints.js +1 -1
- package/dist/application/ProcessCli.js +17 -5
- package/dist/application/ProcessHarness.js +19 -26
- package/dist/application/ProcessNormalization.js +17 -2
- package/dist/application/ProcessOwnership.js +44 -23
- package/dist/application/ProcessSampling.js +111 -25
- package/dist/application/ProjectPermissionStore.js +107 -3
- package/dist/application/ReferenceSourceImport.js +13 -16
- package/dist/application/ReferenceSourceImportPolicy.js +3 -2
- package/dist/application/Setup.js +89 -30
- package/dist/application/SetupClients.js +7 -4
- package/dist/application/SetupInstallFailure.js +24 -0
- package/dist/application/SetupPlan.js +11 -7
- package/dist/application/SetupSkill.js +20 -3
- package/dist/application/TerminalRenderer.js +1 -1
- package/dist/application/Uninstall.js +20 -7
- package/dist/application/Upgrade.js +14 -3
- package/dist/artifacts/MachOSliceArtifactReader.js +1 -1
- package/dist/artifacts/NativeDmgArtifactReader.js +1 -1
- package/dist/artifacts/SafeOutputTree.js +3 -3
- package/dist/browser/CdpBrowserProvider.js +261 -0
- package/dist/browser/CdpCaptureCompleteness.js +90 -0
- package/dist/browser/CdpCaptureDocuments.js +299 -0
- package/dist/browser/CdpCaptureEvents.js +547 -0
- package/dist/browser/CdpCaptureStorage.js +53 -0
- package/dist/browser/CdpCaptureValues.js +68 -0
- package/dist/browser/CdpConnection.js +241 -0
- package/dist/browser/CdpElectronInspection.js +330 -0
- package/dist/browser/CdpElectronProvider.js +134 -0
- package/dist/browser/CdpEndpoint.js +173 -0
- package/dist/browser/CdpObservationSession.js +301 -0
- package/dist/browser/CdpOptionalCommand.js +15 -0
- package/dist/browser/CdpPageCapture.js +376 -0
- package/dist/browser/CdpSafeMetadata.js +203 -0
- package/dist/browser/CdpScreenshot.js +101 -0
- package/dist/browser/CdpTargetSession.js +47 -0
- package/dist/browser/CdpWebMcpDiscovery.js +242 -0
- package/dist/browser/ElectronFileScope.js +50 -0
- package/dist/browser/PngVisualDiff.js +177 -0
- package/dist/browser/SensitiveTextCapture.js +25 -0
- package/dist/browser/WebSourceMapFetcher.js +357 -0
- package/dist/catalogIdentity.js +10 -0
- package/dist/cli.js +6 -0
- package/dist/cliBrowserAdvancedCommands.js +169 -0
- package/dist/cliBrowserCommands.js +312 -0
- package/dist/cliElectronCommands.js +114 -0
- package/dist/cliInvestigationCommands.js +26 -34
- package/dist/cliPolicyCommands.js +60 -12
- package/dist/config.js +80 -1
- package/dist/contracts/browserToolContracts.js +381 -0
- package/dist/contracts/electronToolContracts.js +73 -0
- package/dist/contracts/errorSchemas.js +2 -0
- package/dist/contracts/toolContracts.js +5 -5
- package/dist/domain/browserCompleteness.js +101 -0
- package/dist/domain/browserObservation.js +587 -0
- package/dist/domain/browserSession.js +63 -0
- package/dist/domain/changedBehavior.js +4 -15
- package/dist/domain/electronObservation.js +162 -0
- package/dist/domain/errors.js +41 -0
- package/dist/domain/investigationWorkspace.js +4 -0
- package/dist/domain/jsonLimits.js +26 -0
- package/dist/domain/jsonShape.js +125 -0
- package/dist/domain/permissionPolicy.js +6 -2
- package/dist/domain/processCapture.js +3 -1
- package/dist/domain/processComparison.js +22 -34
- package/dist/domain/reconstructionVerification.js +2 -15
- package/dist/domain/reconstructionVerificationSchemas.js +2 -0
- package/dist/domain/referenceSourceGraph.js +3 -1
- package/dist/domain/staticRuntimeCorrelation.js +2 -0
- package/dist/domain/symbolAnalysis.js +1 -1
- package/dist/domain/webBundleAnalysis.js +195 -0
- package/dist/domain/webBundleAnalyzer.js +423 -0
- package/dist/domain/webCaptureDiff.js +184 -0
- package/dist/domain/webContentArtifact.js +47 -0
- package/dist/domain/webInventory.js +61 -0
- package/dist/domain/webMcpDiscovery.js +68 -0
- package/dist/domain/webScreenshot.js +114 -0
- package/dist/generatedPackageMetadata.js +2 -2
- package/dist/hopper/BridgeLauncher.js +2 -3
- package/dist/hopper/HopperClient.js +51 -11
- package/dist/hopper/HopperProvider.js +6 -1
- package/dist/main.js +42 -31
- package/dist/native/CommandRunner.js +11 -4
- package/dist/native/NativeMacOSProvider.js +3 -2
- package/dist/native/NativeMachoInspection.js +21 -6
- package/dist/server/createServer.js +14 -0
- package/dist/server/registerBrowserTools.js +95 -0
- package/dist/server/registerElectronTools.js +36 -0
- package/dist/server/registerInvestigationTools.js +31 -37
- package/dist/server/registerProcessComparisonTool.js +2 -0
- package/dist/server/registerSessionTools.js +16 -2
- package/dist/server/runDerivedOperation.js +10 -2
- package/package.json +6 -1
- package/scripts/rea.mjs +2 -2
- package/skills/rea-analysis/SKILL.md +28 -3
|
@@ -0,0 +1,547 @@
|
|
|
1
|
+
import { inferJsonShape } from "../domain/jsonShape.js";
|
|
2
|
+
import { CdpCaptureCompleteness } from "./CdpCaptureCompleteness.js";
|
|
3
|
+
import { safeResponseMetadata } from "./CdpSafeMetadata.js";
|
|
4
|
+
import { boundedSensitiveText } from "./SensitiveTextCapture.js";
|
|
5
|
+
import { allowedSanitizedUrl, boundedText, isHttpUrl, numberValue, recordValue, recordsValue, stringValue, } from "./CdpCaptureValues.js";
|
|
6
|
+
/** Bounded event accumulator that drops payload values at ingestion time. */
|
|
7
|
+
export class CdpCaptureEvents {
|
|
8
|
+
input;
|
|
9
|
+
allowedOrigins;
|
|
10
|
+
scripts = new Map();
|
|
11
|
+
network = new Map();
|
|
12
|
+
allowedWebSockets = new Set();
|
|
13
|
+
console = [];
|
|
14
|
+
websockets = [];
|
|
15
|
+
responseMetadata = [];
|
|
16
|
+
agentHints = [];
|
|
17
|
+
completeness = new CdpCaptureCompleteness([
|
|
18
|
+
"network_requests",
|
|
19
|
+
"console_events",
|
|
20
|
+
"websocket_connections",
|
|
21
|
+
"websocket_frames",
|
|
22
|
+
"metadata",
|
|
23
|
+
]);
|
|
24
|
+
originViolation = false;
|
|
25
|
+
navigationDuringCapture = false;
|
|
26
|
+
#mainFrameId;
|
|
27
|
+
#consoleTextBytes = 0;
|
|
28
|
+
#jsonBodyBytes = 0;
|
|
29
|
+
#websocketShapeBytes = 0;
|
|
30
|
+
constructor(input, allowedOrigins) {
|
|
31
|
+
this.input = input;
|
|
32
|
+
this.allowedOrigins = allowedOrigins;
|
|
33
|
+
}
|
|
34
|
+
beginAuthorizedFrame(frameId) {
|
|
35
|
+
this.#mainFrameId = frameId;
|
|
36
|
+
this.originViolation = false;
|
|
37
|
+
this.navigationDuringCapture = false;
|
|
38
|
+
this.scripts.clear();
|
|
39
|
+
this.network.clear();
|
|
40
|
+
this.allowedWebSockets.clear();
|
|
41
|
+
this.console.length = 0;
|
|
42
|
+
this.websockets.length = 0;
|
|
43
|
+
this.responseMetadata.length = 0;
|
|
44
|
+
this.agentHints.length = 0;
|
|
45
|
+
this.#consoleTextBytes = 0;
|
|
46
|
+
this.#jsonBodyBytes = 0;
|
|
47
|
+
this.#websocketShapeBytes = 0;
|
|
48
|
+
this.completeness.reset();
|
|
49
|
+
if (!this.input.include_console_text)
|
|
50
|
+
this.completeness.exclude("console_text", "not_approved", null);
|
|
51
|
+
if (!this.input.include_json_body_shapes)
|
|
52
|
+
this.completeness.exclude("json_body_shapes", "not_approved", null);
|
|
53
|
+
if (!this.input.include_websocket_shapes)
|
|
54
|
+
this.completeness.exclude("websocket_shapes", "not_approved", null);
|
|
55
|
+
}
|
|
56
|
+
beginFinalCapture(frameId) {
|
|
57
|
+
this.#mainFrameId = frameId;
|
|
58
|
+
this.navigationDuringCapture = false;
|
|
59
|
+
}
|
|
60
|
+
ingest(event) {
|
|
61
|
+
const params = recordValue(event.params);
|
|
62
|
+
if (params === undefined)
|
|
63
|
+
return;
|
|
64
|
+
switch (event.method) {
|
|
65
|
+
case "Debugger.scriptParsed":
|
|
66
|
+
this.#script(params);
|
|
67
|
+
break;
|
|
68
|
+
case "Network.requestWillBeSent":
|
|
69
|
+
this.#request(params);
|
|
70
|
+
break;
|
|
71
|
+
case "Network.responseReceived":
|
|
72
|
+
this.#response(params);
|
|
73
|
+
break;
|
|
74
|
+
case "Network.loadingFinished":
|
|
75
|
+
this.#finished(params);
|
|
76
|
+
break;
|
|
77
|
+
case "Runtime.consoleAPICalled":
|
|
78
|
+
this.#console(params);
|
|
79
|
+
break;
|
|
80
|
+
case "Page.frameNavigated":
|
|
81
|
+
this.#frameNavigated(params);
|
|
82
|
+
break;
|
|
83
|
+
case "Network.webSocketCreated":
|
|
84
|
+
this.#websocketCreated(params);
|
|
85
|
+
break;
|
|
86
|
+
case "Network.webSocketFrameSent":
|
|
87
|
+
this.#websocket(params, "sent");
|
|
88
|
+
break;
|
|
89
|
+
case "Network.webSocketFrameReceived":
|
|
90
|
+
this.#websocket(params, "received");
|
|
91
|
+
break;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
#script(params) {
|
|
95
|
+
const scriptId = stringValue(params.scriptId);
|
|
96
|
+
const rawUrl = stringValue(params.url) ?? "";
|
|
97
|
+
const sanitized = allowedSanitizedUrl(rawUrl, this.allowedOrigins);
|
|
98
|
+
if (scriptId === undefined || scriptId.length > 256) {
|
|
99
|
+
this.completeness.exclude("scripts", "invalid_protocol_value");
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
if (sanitized === undefined) {
|
|
103
|
+
this.completeness.exclude("scripts", exclusionReasonForUrl(rawUrl));
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
if (this.scripts.size >= this.input.limits.max_scripts &&
|
|
107
|
+
!this.scripts.has(scriptId)) {
|
|
108
|
+
this.completeness.drop("scripts");
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
const sourceMap = this.#sourceMap(params.sourceMapURL, rawUrl);
|
|
112
|
+
this.scripts.set(scriptId, {
|
|
113
|
+
scriptId,
|
|
114
|
+
rawUrl,
|
|
115
|
+
url: sanitized.url,
|
|
116
|
+
origin: sanitized.origin,
|
|
117
|
+
hash: (stringValue(params.hash) ?? "").slice(0, 512),
|
|
118
|
+
length: Math.max(0, Math.min(Number.MAX_SAFE_INTEGER, Math.trunc(numberValue(params.length) ?? 0))),
|
|
119
|
+
isModule: params.isModule === true,
|
|
120
|
+
language: boundedText(params.scriptLanguage, 100),
|
|
121
|
+
sourceMapUrl: sourceMap.sanitized,
|
|
122
|
+
sourceMapRawUrl: sourceMap.raw,
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
#sourceMap(value, scriptUrl) {
|
|
126
|
+
const declaredUrl = stringValue(value);
|
|
127
|
+
if (declaredUrl === undefined || declaredUrl === "")
|
|
128
|
+
return { sanitized: null, raw: null };
|
|
129
|
+
let rawUrl;
|
|
130
|
+
try {
|
|
131
|
+
rawUrl = new URL(declaredUrl, scriptUrl).href;
|
|
132
|
+
}
|
|
133
|
+
catch {
|
|
134
|
+
this.completeness.exclude("source_maps", "unsupported_url");
|
|
135
|
+
return { sanitized: null, raw: null };
|
|
136
|
+
}
|
|
137
|
+
const sanitized = allowedSanitizedUrl(rawUrl, this.allowedOrigins);
|
|
138
|
+
if (sanitized !== undefined)
|
|
139
|
+
return { sanitized: sanitized.url, raw: rawUrl };
|
|
140
|
+
this.completeness.exclude("source_maps", exclusionReasonForUrl(rawUrl));
|
|
141
|
+
return { sanitized: null, raw: null };
|
|
142
|
+
}
|
|
143
|
+
#request(params) {
|
|
144
|
+
const requestId = stringValue(params.requestId);
|
|
145
|
+
const request = recordValue(params.request);
|
|
146
|
+
if (requestId === undefined || requestId.length > 256) {
|
|
147
|
+
this.completeness.exclude("network_requests", "invalid_protocol_value");
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
if (request === undefined) {
|
|
151
|
+
this.completeness.exclude("network_requests", "invalid_protocol_value");
|
|
152
|
+
this.network.delete(requestId);
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
const sanitized = allowedSanitizedUrl(request.url, this.allowedOrigins);
|
|
156
|
+
if (sanitized === undefined) {
|
|
157
|
+
this.completeness.exclude("network_requests", exclusionReasonForUrl(stringValue(request.url)));
|
|
158
|
+
this.network.delete(requestId);
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
if (this.network.size >= this.input.limits.max_network_events &&
|
|
162
|
+
!this.network.has(requestId)) {
|
|
163
|
+
this.completeness.drop("network_requests");
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
const initiator = recordValue(params.initiator);
|
|
167
|
+
const initiatorFrame = initiatorLocation(initiator);
|
|
168
|
+
const rawInitiatorUrl = stringValue(initiatorFrame?.url);
|
|
169
|
+
const initiatorUrl = allowedSanitizedUrl(rawInitiatorUrl, this.allowedOrigins);
|
|
170
|
+
if (rawInitiatorUrl !== undefined && initiatorUrl === undefined)
|
|
171
|
+
this.completeness.exclude("network_initiators", exclusionReasonForUrl(rawInitiatorUrl));
|
|
172
|
+
this.network.set(requestId, {
|
|
173
|
+
request_id: requestId,
|
|
174
|
+
url: sanitized.url,
|
|
175
|
+
origin: sanitized.origin ?? "",
|
|
176
|
+
method: (stringValue(request.method) ?? "GET").slice(0, 32),
|
|
177
|
+
resource_type: boundedText(params.type, 100),
|
|
178
|
+
status: null,
|
|
179
|
+
mime_type: null,
|
|
180
|
+
encoded_data_length: null,
|
|
181
|
+
initiator: {
|
|
182
|
+
type: (stringValue(initiator?.type) ?? "other").slice(0, 100),
|
|
183
|
+
url: initiatorUrl?.url ?? null,
|
|
184
|
+
line: integerOrNull(initiatorFrame?.lineNumber),
|
|
185
|
+
column: integerOrNull(initiatorFrame?.columnNumber),
|
|
186
|
+
},
|
|
187
|
+
body_shapes: this.#requestBodyShape(request),
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
#response(params) {
|
|
191
|
+
const requestId = stringValue(params.requestId);
|
|
192
|
+
if (requestId === undefined || requestId.length > 256) {
|
|
193
|
+
this.completeness.exclude("network_requests", "invalid_protocol_value");
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
const current = this.network.get(requestId);
|
|
197
|
+
const response = recordValue(params.response);
|
|
198
|
+
if (current === undefined)
|
|
199
|
+
return;
|
|
200
|
+
const sanitized = allowedSanitizedUrl(response?.url, this.allowedOrigins);
|
|
201
|
+
if (response === undefined || sanitized === undefined) {
|
|
202
|
+
this.completeness.exclude("network_requests", response === undefined
|
|
203
|
+
? "invalid_protocol_value"
|
|
204
|
+
: exclusionReasonForUrl(stringValue(response.url)));
|
|
205
|
+
this.network.delete(requestId);
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
this.network.set(requestId, {
|
|
209
|
+
...current,
|
|
210
|
+
status: numberValue(response.status) ?? null,
|
|
211
|
+
mime_type: boundedText(response.mimeType, 256),
|
|
212
|
+
});
|
|
213
|
+
if (this.responseMetadata.length >= this.input.limits.max_network_events) {
|
|
214
|
+
this.completeness.truncate("metadata");
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
const metadata = safeResponseMetadata(requestId, sanitized.url, response, this.allowedOrigins);
|
|
218
|
+
this.responseMetadata.push(metadata.response);
|
|
219
|
+
this.agentHints.push(...metadata.agentHints);
|
|
220
|
+
}
|
|
221
|
+
#finished(params) {
|
|
222
|
+
const requestId = stringValue(params.requestId);
|
|
223
|
+
if (requestId === undefined || requestId.length > 256)
|
|
224
|
+
return;
|
|
225
|
+
const current = this.network.get(requestId);
|
|
226
|
+
if (current === undefined)
|
|
227
|
+
return;
|
|
228
|
+
this.network.set(requestId, {
|
|
229
|
+
...current,
|
|
230
|
+
encoded_data_length: Math.max(0, numberValue(params.encodedDataLength) ?? 0),
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
#console(params) {
|
|
234
|
+
const frame = firstCallFrame(recordValue(params.stackTrace));
|
|
235
|
+
if (frame === undefined) {
|
|
236
|
+
this.completeness.exclude("console_events", "unattributed_origin");
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
const source = allowedSanitizedUrl(frame.url, this.allowedOrigins);
|
|
240
|
+
if (source === undefined) {
|
|
241
|
+
this.completeness.exclude("console_events", exclusionReasonForUrl(stringValue(frame.url)));
|
|
242
|
+
return;
|
|
243
|
+
}
|
|
244
|
+
if (this.console.length >= this.input.limits.max_console_events) {
|
|
245
|
+
this.completeness.drop("console_events");
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
const arguments_ = recordsValue(params.args);
|
|
249
|
+
this.console.push({
|
|
250
|
+
type: (stringValue(params.type) ?? "unknown").slice(0, 100),
|
|
251
|
+
timestamp: numberValue(params.timestamp) ?? 0,
|
|
252
|
+
argument_types: arguments_
|
|
253
|
+
.map((argument) => (stringValue(argument.type) ?? "unknown").slice(0, 100))
|
|
254
|
+
.slice(0, 100),
|
|
255
|
+
url: source.url,
|
|
256
|
+
line: integerOrNull(frame.lineNumber),
|
|
257
|
+
column: integerOrNull(frame.columnNumber),
|
|
258
|
+
text_capture: this.#consoleText(arguments_),
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
#websocket(params, direction) {
|
|
262
|
+
const requestId = stringValue(params.requestId);
|
|
263
|
+
if (requestId === undefined ||
|
|
264
|
+
requestId.length > 256 ||
|
|
265
|
+
(!this.network.has(requestId) && !this.allowedWebSockets.has(requestId)))
|
|
266
|
+
return;
|
|
267
|
+
if (this.websockets.length >= this.input.limits.max_websocket_events) {
|
|
268
|
+
this.completeness.drop("websocket_frames");
|
|
269
|
+
return;
|
|
270
|
+
}
|
|
271
|
+
const response = recordValue(params.response);
|
|
272
|
+
const payload = stringValue(response?.payloadData) ?? "";
|
|
273
|
+
const opcode = Math.max(0, Math.trunc(numberValue(response?.opcode) ?? 0));
|
|
274
|
+
const decoded = opcode === 1 ? undefined : decodeBase64(payload);
|
|
275
|
+
if (opcode !== 1 && decoded === undefined)
|
|
276
|
+
this.completeness.exclude("websocket_frames", "invalid_protocol_value");
|
|
277
|
+
this.websockets.push({
|
|
278
|
+
request_id: requestId,
|
|
279
|
+
direction,
|
|
280
|
+
opcode,
|
|
281
|
+
payload_bytes: opcode === 1 ? Buffer.byteLength(payload) : (decoded?.byteLength ?? 0),
|
|
282
|
+
payload_shape: this.#websocketShape(payload, opcode),
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
responseBodyRequestIds() {
|
|
286
|
+
if (!this.input.include_json_body_shapes)
|
|
287
|
+
return [];
|
|
288
|
+
return [...this.network.values()]
|
|
289
|
+
.filter((request) => isJsonMediaType(request.mime_type))
|
|
290
|
+
.map((request) => request.request_id);
|
|
291
|
+
}
|
|
292
|
+
responseBodyUnavailable(requestId) {
|
|
293
|
+
this.#updateResponseShape(requestId, null, false);
|
|
294
|
+
this.completeness.unavailable("json_body_shapes");
|
|
295
|
+
}
|
|
296
|
+
ingestResponseBody(requestId, value) {
|
|
297
|
+
const result = recordValue(value);
|
|
298
|
+
const body = stringValue(result?.body);
|
|
299
|
+
if (body === undefined) {
|
|
300
|
+
this.#invalidResponseBody(requestId);
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
const decoded = result?.base64Encoded === true
|
|
304
|
+
? decodeBase64(body)?.toString("utf8")
|
|
305
|
+
: body;
|
|
306
|
+
if (decoded === undefined) {
|
|
307
|
+
this.#invalidResponseBody(requestId);
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
310
|
+
const inferred = this.#inferBodyShape(decoded);
|
|
311
|
+
this.#updateResponseShape(requestId, inferred.shape, inferred.truncated);
|
|
312
|
+
}
|
|
313
|
+
#invalidResponseBody(requestId) {
|
|
314
|
+
this.#updateResponseShape(requestId, null, false);
|
|
315
|
+
this.completeness.exclude("json_body_shapes", "invalid_protocol_value");
|
|
316
|
+
}
|
|
317
|
+
#requestBodyShape(request) {
|
|
318
|
+
if (!this.input.include_json_body_shapes)
|
|
319
|
+
return { status: "not_approved", request: null, response: null };
|
|
320
|
+
const body = stringValue(request.postData);
|
|
321
|
+
if (!isJsonContentType(recordValue(request.headers)) || body === undefined)
|
|
322
|
+
return { status: "unavailable", request: null, response: null };
|
|
323
|
+
const inferred = this.#inferBodyShape(body);
|
|
324
|
+
return {
|
|
325
|
+
status: inferred.truncated
|
|
326
|
+
? "truncated"
|
|
327
|
+
: inferred.shape === null
|
|
328
|
+
? "unavailable"
|
|
329
|
+
: "included",
|
|
330
|
+
request: inferred.shape,
|
|
331
|
+
response: null,
|
|
332
|
+
};
|
|
333
|
+
}
|
|
334
|
+
#inferBodyShape(text) {
|
|
335
|
+
const bytes = Buffer.byteLength(text);
|
|
336
|
+
const remaining = this.input.limits.max_total_json_body_bytes - this.#jsonBodyBytes;
|
|
337
|
+
if (bytes > this.input.limits.max_json_body_bytes || bytes > remaining) {
|
|
338
|
+
this.completeness.truncate("json_body_shapes");
|
|
339
|
+
return { shape: null, truncated: true };
|
|
340
|
+
}
|
|
341
|
+
this.#jsonBodyBytes += bytes;
|
|
342
|
+
const shape = inferJsonShape(text, {
|
|
343
|
+
maximumBytes: this.input.limits.max_json_body_bytes,
|
|
344
|
+
maximumNodes: this.input.limits.max_json_shape_nodes,
|
|
345
|
+
maximumDepth: this.input.limits.max_json_shape_depth,
|
|
346
|
+
});
|
|
347
|
+
if (shape?.truncated === true)
|
|
348
|
+
this.completeness.truncate("json_body_shapes");
|
|
349
|
+
return { shape, truncated: shape?.truncated === true };
|
|
350
|
+
}
|
|
351
|
+
#updateResponseShape(requestId, response, truncated) {
|
|
352
|
+
const current = this.network.get(requestId);
|
|
353
|
+
if (current === undefined)
|
|
354
|
+
return;
|
|
355
|
+
const request = current.body_shapes.request;
|
|
356
|
+
const status = truncated || current.body_shapes.status === "truncated"
|
|
357
|
+
? "truncated"
|
|
358
|
+
: response !== null
|
|
359
|
+
? "included"
|
|
360
|
+
: request !== null
|
|
361
|
+
? "partial"
|
|
362
|
+
: "unavailable";
|
|
363
|
+
this.network.set(requestId, {
|
|
364
|
+
...current,
|
|
365
|
+
body_shapes: { status, request, response },
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
#consoleText(arguments_) {
|
|
369
|
+
if (!this.input.include_console_text)
|
|
370
|
+
return {
|
|
371
|
+
status: "not_approved",
|
|
372
|
+
values: [],
|
|
373
|
+
retained_bytes: 0,
|
|
374
|
+
truncated_values: 0,
|
|
375
|
+
};
|
|
376
|
+
const values = [];
|
|
377
|
+
let retainedBytes = 0;
|
|
378
|
+
let truncatedValues = 0;
|
|
379
|
+
for (const [argumentIndex, argument] of arguments_.entries()) {
|
|
380
|
+
const primitive = consolePrimitive(argument);
|
|
381
|
+
if (primitive === undefined)
|
|
382
|
+
continue;
|
|
383
|
+
const remaining = this.input.limits.max_total_console_text_bytes - this.#consoleTextBytes;
|
|
384
|
+
if (remaining <= 0) {
|
|
385
|
+
truncatedValues += 1;
|
|
386
|
+
continue;
|
|
387
|
+
}
|
|
388
|
+
const bounded = boundedSensitiveText(primitive.text, Math.min(this.input.limits.max_console_text_field_bytes, remaining));
|
|
389
|
+
values.push({
|
|
390
|
+
argument_index: argumentIndex,
|
|
391
|
+
type: primitive.type,
|
|
392
|
+
text: bounded.text,
|
|
393
|
+
});
|
|
394
|
+
retainedBytes += bounded.bytes;
|
|
395
|
+
this.#consoleTextBytes += bounded.bytes;
|
|
396
|
+
if (bounded.truncated)
|
|
397
|
+
truncatedValues += 1;
|
|
398
|
+
}
|
|
399
|
+
if (truncatedValues > 0)
|
|
400
|
+
this.completeness.truncate("console_text");
|
|
401
|
+
return {
|
|
402
|
+
status: truncatedValues > 0 ? "truncated" : "included",
|
|
403
|
+
values,
|
|
404
|
+
retained_bytes: retainedBytes,
|
|
405
|
+
truncated_values: truncatedValues,
|
|
406
|
+
};
|
|
407
|
+
}
|
|
408
|
+
#websocketShape(payload, opcode) {
|
|
409
|
+
if (!this.input.include_websocket_shapes)
|
|
410
|
+
return null;
|
|
411
|
+
if (opcode !== 1)
|
|
412
|
+
return { format: "binary", json_shape: null, truncated: false };
|
|
413
|
+
const bytes = Buffer.byteLength(payload);
|
|
414
|
+
const remaining = this.input.limits.max_total_websocket_shape_bytes -
|
|
415
|
+
this.#websocketShapeBytes;
|
|
416
|
+
if (bytes > this.input.limits.max_websocket_shape_bytes ||
|
|
417
|
+
bytes > remaining) {
|
|
418
|
+
this.completeness.truncate("websocket_shapes");
|
|
419
|
+
return { format: "text", json_shape: null, truncated: true };
|
|
420
|
+
}
|
|
421
|
+
this.#websocketShapeBytes += bytes;
|
|
422
|
+
const shape = inferJsonShape(payload, {
|
|
423
|
+
maximumBytes: this.input.limits.max_websocket_shape_bytes,
|
|
424
|
+
maximumNodes: this.input.limits.max_json_shape_nodes,
|
|
425
|
+
maximumDepth: this.input.limits.max_json_shape_depth,
|
|
426
|
+
});
|
|
427
|
+
if (shape?.truncated === true)
|
|
428
|
+
this.completeness.truncate("websocket_shapes");
|
|
429
|
+
return {
|
|
430
|
+
format: shape === null ? "text" : "json",
|
|
431
|
+
json_shape: shape,
|
|
432
|
+
truncated: shape?.truncated === true,
|
|
433
|
+
};
|
|
434
|
+
}
|
|
435
|
+
#websocketCreated(params) {
|
|
436
|
+
const requestId = stringValue(params.requestId);
|
|
437
|
+
const rawUrl = stringValue(params.url);
|
|
438
|
+
if (requestId === undefined || requestId.length > 256) {
|
|
439
|
+
this.completeness.exclude("websocket_connections", "invalid_protocol_value");
|
|
440
|
+
return;
|
|
441
|
+
}
|
|
442
|
+
if (rawUrl === undefined) {
|
|
443
|
+
this.completeness.exclude("websocket_connections", "invalid_protocol_value");
|
|
444
|
+
return;
|
|
445
|
+
}
|
|
446
|
+
let parsed;
|
|
447
|
+
try {
|
|
448
|
+
parsed = new URL(rawUrl);
|
|
449
|
+
}
|
|
450
|
+
catch {
|
|
451
|
+
this.completeness.exclude("websocket_connections", "unsupported_url");
|
|
452
|
+
return;
|
|
453
|
+
}
|
|
454
|
+
if (parsed.protocol === "ws:")
|
|
455
|
+
parsed.protocol = "http:";
|
|
456
|
+
else if (parsed.protocol === "wss:")
|
|
457
|
+
parsed.protocol = "https:";
|
|
458
|
+
else {
|
|
459
|
+
this.completeness.exclude("websocket_connections", "unsupported_url");
|
|
460
|
+
return;
|
|
461
|
+
}
|
|
462
|
+
if (!this.allowedOrigins.has(parsed.origin)) {
|
|
463
|
+
this.completeness.exclude("websocket_connections", "disallowed_origin");
|
|
464
|
+
return;
|
|
465
|
+
}
|
|
466
|
+
if (!this.allowedWebSockets.has(requestId) &&
|
|
467
|
+
this.allowedWebSockets.size >= this.input.limits.max_websocket_events) {
|
|
468
|
+
this.completeness.drop("websocket_connections");
|
|
469
|
+
return;
|
|
470
|
+
}
|
|
471
|
+
this.allowedWebSockets.add(requestId);
|
|
472
|
+
}
|
|
473
|
+
#frameNavigated(params) {
|
|
474
|
+
const frame = recordValue(params.frame);
|
|
475
|
+
if (stringValue(frame?.id) !== this.#mainFrameId)
|
|
476
|
+
return;
|
|
477
|
+
this.navigationDuringCapture = true;
|
|
478
|
+
const rawUrl = stringValue(frame?.url);
|
|
479
|
+
if (isHttpUrl(rawUrl) &&
|
|
480
|
+
allowedSanitizedUrl(rawUrl, this.allowedOrigins) === undefined)
|
|
481
|
+
this.originViolation = true;
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
const firstCallFrame = (value) => recordsValue(value?.callFrames)[0];
|
|
485
|
+
const initiatorLocation = (initiator) => {
|
|
486
|
+
let stack = recordValue(initiator?.stack);
|
|
487
|
+
for (let depth = 0; depth < 32 && stack !== undefined; depth += 1) {
|
|
488
|
+
const frame = firstCallFrame(stack);
|
|
489
|
+
if (frame !== undefined)
|
|
490
|
+
return frame;
|
|
491
|
+
stack = recordValue(stack.parent);
|
|
492
|
+
}
|
|
493
|
+
return stringValue(initiator?.url) === undefined ? undefined : initiator;
|
|
494
|
+
};
|
|
495
|
+
const exclusionReasonForUrl = (value) => value === undefined || value === ""
|
|
496
|
+
? "unattributed_origin"
|
|
497
|
+
: isHttpUrl(value)
|
|
498
|
+
? "disallowed_origin"
|
|
499
|
+
: "unsupported_url";
|
|
500
|
+
const integerOrNull = (value) => {
|
|
501
|
+
const number = numberValue(value);
|
|
502
|
+
return number === undefined ? null : Math.max(0, Math.trunc(number));
|
|
503
|
+
};
|
|
504
|
+
const isJsonContentType = (headers) => {
|
|
505
|
+
if (headers === undefined)
|
|
506
|
+
return false;
|
|
507
|
+
for (const [name, value] of Object.entries(headers))
|
|
508
|
+
if (name.toLowerCase() === "content-type" &&
|
|
509
|
+
isJsonMediaType(stringValue(value)))
|
|
510
|
+
return true;
|
|
511
|
+
return false;
|
|
512
|
+
};
|
|
513
|
+
const isJsonMediaType = (value) => {
|
|
514
|
+
const mediaType = value?.split(";", 1)[0]?.trim().toLowerCase();
|
|
515
|
+
return (mediaType === "application/json" || mediaType?.endsWith("+json") === true);
|
|
516
|
+
};
|
|
517
|
+
const consolePrimitive = (value) => {
|
|
518
|
+
const type = stringValue(value.type);
|
|
519
|
+
switch (type) {
|
|
520
|
+
case "string":
|
|
521
|
+
return typeof value.value === "string"
|
|
522
|
+
? { type, text: value.value }
|
|
523
|
+
: undefined;
|
|
524
|
+
case "boolean":
|
|
525
|
+
return typeof value.value === "boolean"
|
|
526
|
+
? { type, text: String(value.value) }
|
|
527
|
+
: undefined;
|
|
528
|
+
case "number":
|
|
529
|
+
return typeof value.value === "number"
|
|
530
|
+
? { type, text: String(value.value) }
|
|
531
|
+
: typeof value.unserializableValue === "string"
|
|
532
|
+
? { type, text: value.unserializableValue }
|
|
533
|
+
: undefined;
|
|
534
|
+
case "bigint":
|
|
535
|
+
return typeof value.unserializableValue === "string"
|
|
536
|
+
? { type, text: value.unserializableValue }
|
|
537
|
+
: undefined;
|
|
538
|
+
case "undefined":
|
|
539
|
+
return { type, text: "undefined" };
|
|
540
|
+
default:
|
|
541
|
+
return undefined;
|
|
542
|
+
}
|
|
543
|
+
};
|
|
544
|
+
const BASE64 = /^(?:[A-Za-z\d+/]{4})*(?:[A-Za-z\d+/]{2}==|[A-Za-z\d+/]{3}=)?$/u;
|
|
545
|
+
const decodeBase64 = (value) => value.length % 4 === 0 && BASE64.test(value)
|
|
546
|
+
? Buffer.from(value, "base64")
|
|
547
|
+
: undefined;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { CdpConnection } from "./CdpConnection.js";
|
|
2
|
+
import { optionalCdpCommand } from "./CdpOptionalCommand.js";
|
|
3
|
+
import { numberValue, recordValue, recordsValue, stringValue, } from "./CdpCaptureValues.js";
|
|
4
|
+
/** Capture redacted, bounded storage metadata for one authorized origin. */
|
|
5
|
+
export const captureStorage = async (context, origin, limitations) => {
|
|
6
|
+
const quota = recordValue(await optionalCdpCommand(context, "Storage.getUsageAndQuota", { origin }, limitations));
|
|
7
|
+
const local = context.input.include_storage_keys
|
|
8
|
+
? await storageKeys(context, origin, true, limitations)
|
|
9
|
+
: [];
|
|
10
|
+
const session = context.input.include_storage_keys
|
|
11
|
+
? await storageKeys(context, origin, false, limitations)
|
|
12
|
+
: [];
|
|
13
|
+
const indexed = context.input.include_storage_keys
|
|
14
|
+
? stringArray(recordValue(await optionalCdpCommand(context, "IndexedDB.requestDatabaseNames", { securityOrigin: origin }, limitations))?.databaseNames)
|
|
15
|
+
: [];
|
|
16
|
+
const caches = context.input.include_storage_keys
|
|
17
|
+
? recordsValue(recordValue(await optionalCdpCommand(context, "CacheStorage.requestCacheNames", { securityOrigin: origin }, limitations))?.caches).flatMap((cache) => {
|
|
18
|
+
const name = stringValue(cache.cacheName);
|
|
19
|
+
return name === undefined ? [] : [name.slice(0, 1_024)];
|
|
20
|
+
})
|
|
21
|
+
: [];
|
|
22
|
+
const maximum = context.input.limits.max_storage_keys;
|
|
23
|
+
return {
|
|
24
|
+
value: {
|
|
25
|
+
origin,
|
|
26
|
+
usage_bytes: numberValue(quota?.usage) ?? null,
|
|
27
|
+
quota_bytes: numberValue(quota?.quota) ?? null,
|
|
28
|
+
local_storage_keys: local.slice(0, maximum),
|
|
29
|
+
session_storage_keys: session.slice(0, maximum),
|
|
30
|
+
indexed_db_names: indexed.slice(0, maximum),
|
|
31
|
+
cache_names: caches.slice(0, maximum),
|
|
32
|
+
values_redacted: true,
|
|
33
|
+
},
|
|
34
|
+
truncated: [local, session, indexed, caches].some((items) => items.length > maximum),
|
|
35
|
+
};
|
|
36
|
+
};
|
|
37
|
+
const storageKeys = async (context, origin, isLocalStorage, limitations) => {
|
|
38
|
+
const result = recordValue(await optionalCdpCommand(context, "DOMStorage.getDOMStorageItems", { storageId: { securityOrigin: origin, isLocalStorage } }, limitations));
|
|
39
|
+
return Array.isArray(result?.entries)
|
|
40
|
+
? result.entries.flatMap((entry) => {
|
|
41
|
+
if (!Array.isArray(entry))
|
|
42
|
+
return [];
|
|
43
|
+
const key = stringValue(entry[0]);
|
|
44
|
+
return key === undefined ? [] : [key.slice(0, 1_024)];
|
|
45
|
+
})
|
|
46
|
+
: [];
|
|
47
|
+
};
|
|
48
|
+
const stringArray = (value) => Array.isArray(value)
|
|
49
|
+
? value.flatMap((item) => {
|
|
50
|
+
const text = stringValue(item);
|
|
51
|
+
return text === undefined ? [] : [text.slice(0, 1_024)];
|
|
52
|
+
})
|
|
53
|
+
: [];
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { AnalysisCancelledError, BrowserObservationError, } from "../domain/errors.js";
|
|
2
|
+
import { sanitizeBrowserUrl } from "../domain/browserObservation.js";
|
|
3
|
+
import { createWebTextArtifact } from "../domain/webContentArtifact.js";
|
|
4
|
+
const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
5
|
+
export const recordValue = (value) => isRecord(value) ? value : undefined;
|
|
6
|
+
export const recordsValue = (value) => Array.isArray(value)
|
|
7
|
+
? value.flatMap((item) => {
|
|
8
|
+
const record = recordValue(item);
|
|
9
|
+
return record === undefined ? [] : [record];
|
|
10
|
+
})
|
|
11
|
+
: [];
|
|
12
|
+
export const stringValue = (value) => typeof value === "string" ? value : undefined;
|
|
13
|
+
export const numberValue = (value) => typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
14
|
+
export const requiredRecord = (value) => {
|
|
15
|
+
const record = recordValue(value);
|
|
16
|
+
if (record === undefined)
|
|
17
|
+
throw new BrowserObservationError("inspect_web_page", "protocol_error");
|
|
18
|
+
return record;
|
|
19
|
+
};
|
|
20
|
+
export const boundedText = (value, maximum = 1_024) => {
|
|
21
|
+
const text = stringValue(value);
|
|
22
|
+
return text === undefined ? null : text.slice(0, maximum);
|
|
23
|
+
};
|
|
24
|
+
export const isHttpUrl = (value) => {
|
|
25
|
+
if (value === undefined)
|
|
26
|
+
return false;
|
|
27
|
+
try {
|
|
28
|
+
const protocol = new URL(value).protocol;
|
|
29
|
+
return protocol === "http:" || protocol === "https:";
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
};
|
|
35
|
+
export const allowedSanitizedUrl = (value, allowedOrigins) => {
|
|
36
|
+
const text = stringValue(value);
|
|
37
|
+
if (text === undefined)
|
|
38
|
+
return undefined;
|
|
39
|
+
const sanitized = sanitizeBrowserUrl(text);
|
|
40
|
+
return sanitized.origin !== null && allowedOrigins.has(sanitized.origin)
|
|
41
|
+
? sanitized
|
|
42
|
+
: undefined;
|
|
43
|
+
};
|
|
44
|
+
export const sourceResult = (source) => ({
|
|
45
|
+
included: true,
|
|
46
|
+
artifact: createWebTextArtifact(source, "text/javascript"),
|
|
47
|
+
});
|
|
48
|
+
export const sourceExcluded = (reason) => ({ included: false, reason });
|
|
49
|
+
export const delayWithCancellation = async (durationMs, operation, signal) => {
|
|
50
|
+
if (durationMs === 0)
|
|
51
|
+
return;
|
|
52
|
+
if (signal?.aborted === true)
|
|
53
|
+
throw new AnalysisCancelledError(operation);
|
|
54
|
+
await new Promise((resolve, reject) => {
|
|
55
|
+
const timer = setTimeout(() => {
|
|
56
|
+
signal?.removeEventListener("abort", onAbort);
|
|
57
|
+
resolve();
|
|
58
|
+
}, durationMs);
|
|
59
|
+
const onAbort = () => {
|
|
60
|
+
clearTimeout(timer);
|
|
61
|
+
signal?.removeEventListener("abort", onAbort);
|
|
62
|
+
reject(new AnalysisCancelledError(operation));
|
|
63
|
+
};
|
|
64
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
65
|
+
if (signal?.aborted === true)
|
|
66
|
+
onAbort();
|
|
67
|
+
});
|
|
68
|
+
};
|