rea-agents 2.4.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/README.md +91 -19
  2. package/bridge/ghidra/ReaGhidraBridge.java +524 -4
  3. package/bridge/hopper_bridge.py +313 -28
  4. package/dist/application/AnalysisClientCleanup.js +14 -0
  5. package/dist/application/AnalysisContextQueries.js +130 -0
  6. package/dist/application/AnalysisSnapshotCache.js +4 -2
  7. package/dist/application/ApplicationWorkflowEvidenceResolver.js +31 -13
  8. package/dist/application/ArtifactInventory/scanCanonical.js +1 -1
  9. package/dist/application/BinarySession.js +20 -8
  10. package/dist/application/BinarySessionComposition.js +21 -0
  11. package/dist/application/BinarySessionRecords.js +79 -9
  12. package/dist/application/BinarySessionStatus.js +31 -1
  13. package/dist/application/BrowserEvidence.js +14 -0
  14. package/dist/application/BrowserScenarioCapturePort.js +1 -0
  15. package/dist/application/BrowserScenarioCaptureService.js +53 -0
  16. package/dist/application/BrowserScenarioEvidence.js +40 -0
  17. package/dist/application/CallPathTracing.js +160 -0
  18. package/dist/application/CapabilityInventory.js +84 -14
  19. package/dist/application/ClientRegistrationStatus.js +4 -1
  20. package/dist/application/CompositeProvider.js +10 -1
  21. package/dist/application/DirectAnalysis.js +13 -4
  22. package/dist/application/EnhancedLiteralTracing.js +118 -0
  23. package/dist/application/EnhancedTools.js +71 -127
  24. package/dist/application/EvidenceLedger.js +45 -16
  25. package/dist/application/JavaScriptApplicationWorkflowEvidence.js +13 -0
  26. package/dist/application/JavaScriptApplicationWorkflowService.js +27 -1
  27. package/dist/application/JavaScriptArtifactAnalysis.js +13 -4
  28. package/dist/application/JavaScriptRuntimeObservationEvidence.js +34 -0
  29. package/dist/application/JavaScriptRuntimeObservationPort.js +1 -0
  30. package/dist/application/JavaScriptRuntimeObservationService.js +46 -0
  31. package/dist/application/JavaScriptSemanticGraphAsyncProjection.js +147 -0
  32. package/dist/application/JavaScriptSemanticGraphBuilder.js +34 -6
  33. package/dist/application/JavaScriptSemanticGraphChildProcessProjection.js +156 -0
  34. package/dist/application/JavaScriptSemanticGraphConstruction.js +2 -2
  35. package/dist/application/JavaScriptSemanticGraphDataProjection.js +203 -0
  36. package/dist/application/JavaScriptSemanticGraphFingerprintProjection.js +25 -0
  37. package/dist/application/JavaScriptSemanticGraphFlowProjection.js +129 -8
  38. package/dist/application/JavaScriptSemanticGraphObjectProjection.js +81 -0
  39. package/dist/application/JavaScriptSemanticGraphProjection.js +14 -7
  40. package/dist/application/JavaScriptSemanticGraphResourceProjection.js +59 -0
  41. package/dist/application/JavaScriptSemanticGraphValueProjection.js +77 -0
  42. package/dist/application/LazyAnalysisProvider.js +77 -0
  43. package/dist/application/LoopbackReplay.js +1 -1
  44. package/dist/application/NativeApiInspection.js +80 -0
  45. package/dist/application/ProcessReactiveEffects.js +6 -0
  46. package/dist/application/ProcessReactiveHarness.js +16 -1
  47. package/dist/application/ProjectPermissionStore.js +1 -0
  48. package/dist/application/ReconstructionObligationAuthority.js +72 -0
  49. package/dist/application/ReconstructionObligationCandidates.js +265 -0
  50. package/dist/application/ReconstructionObligationLedgerDiagnostics.js +47 -0
  51. package/dist/application/ReconstructionObligationLedgerEvaluation.js +263 -0
  52. package/dist/application/ReconstructionObligationLedgerProjection.js +86 -0
  53. package/dist/application/ReconstructionObligationLedgerService.js +72 -0
  54. package/dist/application/ReconstructionObligationOriginalCases.js +33 -0
  55. package/dist/application/ReconstructionObligationPolicies.js +132 -0
  56. package/dist/application/ReconstructionReadinessService.js +48 -0
  57. package/dist/application/SessionProviderRouter.js +6 -6
  58. package/dist/application/SetupClientConfiguration.js +4 -2
  59. package/dist/application/runtime.js +26 -9
  60. package/dist/artifacts/ArtifactProvider.js +87 -22
  61. package/dist/browser/BrowserScenarioSecrets.js +53 -0
  62. package/dist/browser/BrowserScenarioSessionPort.js +1 -0
  63. package/dist/browser/CdpBrowserProvider.js +2 -2
  64. package/dist/browser/CdpCaptureDocuments.js +57 -0
  65. package/dist/browser/CdpCaptureStorage.js +71 -29
  66. package/dist/browser/CdpCaptureStorageFingerprints.js +328 -0
  67. package/dist/browser/CdpEndpoint.js +8 -6
  68. package/dist/browser/CdpPageCapture.js +2 -0
  69. package/dist/browser/JavaScriptRuntimeScope.js +38 -0
  70. package/dist/browser/PlaywrightBrowserScenarioProvider.js +259 -0
  71. package/dist/browser/PlaywrightExecutionBoundary.js +42 -0
  72. package/dist/browser/PlaywrightScenarioActions.js +62 -0
  73. package/dist/browser/PlaywrightScenarioArtifacts.js +238 -0
  74. package/dist/browser/PlaywrightScenarioBrowser.js +144 -0
  75. package/dist/browser/PlaywrightScenarioEvents.js +301 -0
  76. package/dist/browser/PlaywrightScenarioSession.js +300 -0
  77. package/dist/browser/V8InspectorCaptureProjection.js +110 -0
  78. package/dist/browser/V8InspectorEndpoint.js +72 -0
  79. package/dist/browser/V8InspectorProvider.js +267 -0
  80. package/dist/catalogIdentity.js +11 -3
  81. package/dist/cli/artifactCommands.js +44 -3
  82. package/dist/cli/coreAnalysisCommands.js +78 -0
  83. package/dist/cli.js +4 -0
  84. package/dist/cliApplicationCommands.js +30 -2
  85. package/dist/cliBrowserAdvancedCommands.js +17 -5
  86. package/dist/cliBrowserCommands.js +1 -0
  87. package/dist/cliBrowserScenarioCommands.js +43 -0
  88. package/dist/cliCommandNames.js +9 -0
  89. package/dist/cliJavaScriptRuntimeCommands.js +164 -0
  90. package/dist/cliObservationOptions.js +4 -0
  91. package/dist/cliPolicyCommands.js +1 -0
  92. package/dist/config/environment.js +12 -0
  93. package/dist/config/parseConfig.js +88 -17
  94. package/dist/contracts/analysisSearchContract.js +14 -0
  95. package/dist/contracts/applicationToolContracts.js +86 -2
  96. package/dist/contracts/applicationWorkflowInputContracts.js +18 -0
  97. package/dist/contracts/artifactToolContracts.js +29 -2
  98. package/dist/contracts/browserProviderToolContracts.js +7 -0
  99. package/dist/contracts/browserScenarioToolContracts.js +81 -0
  100. package/dist/contracts/browserToolContracts.js +88 -98
  101. package/dist/contracts/enhancedInputs.js +35 -6
  102. package/dist/contracts/functionInstructionContract.js +8 -0
  103. package/dist/contracts/functionWorkflowToolContracts.js +30 -0
  104. package/dist/contracts/hopperMemoryContracts.js +22 -0
  105. package/dist/contracts/javascriptApplicationWorkflowExamples.js +42 -0
  106. package/dist/contracts/javascriptRuntimeObservationToolContracts.js +58 -0
  107. package/dist/contracts/promptContracts.js +14 -15
  108. package/dist/contracts/reconstructionReadinessExample.js +436 -0
  109. package/dist/contracts/toolContractExamples.js +8 -0
  110. package/dist/contracts/toolContractTypes.js +1 -0
  111. package/dist/contracts/toolContracts.js +34 -25
  112. package/dist/contracts/toolEffects.js +25 -0
  113. package/dist/contracts/toolOutputSchemaGroups.js +120 -30
  114. package/dist/contracts/toolOutputSchemaPrimitives.js +37 -0
  115. package/dist/domain/analysisErrorPresentation.js +8 -2
  116. package/dist/domain/analysisErrorProjection.js +9 -0
  117. package/dist/domain/artifactInspection.js +347 -0
  118. package/dist/domain/browserCaptureComparison.js +82 -0
  119. package/dist/domain/browserObservation.js +8 -1
  120. package/dist/domain/browserObservationErrors.js +1 -0
  121. package/dist/domain/browserObservationSchemas.js +26 -0
  122. package/dist/domain/browserScenario.js +165 -0
  123. package/dist/domain/browserScenarioCapture.js +57 -0
  124. package/dist/domain/browserScenarioCaptureValues.js +204 -0
  125. package/dist/domain/browserScenarioDiff.js +324 -0
  126. package/dist/domain/browserScenarioDiffValues.js +157 -0
  127. package/dist/domain/browserScenarioNormalization.js +45 -0
  128. package/dist/domain/browserScenarioValues.js +372 -0
  129. package/dist/domain/bytecodeProvider.js +179 -0
  130. package/dist/domain/conformancePackage.js +169 -0
  131. package/dist/domain/conformanceReplay.js +104 -0
  132. package/dist/domain/conformanceTrustGate.js +193 -0
  133. package/dist/domain/customProtocolCapture.js +137 -0
  134. package/dist/domain/eventProcessTree.js +214 -0
  135. package/dist/domain/evidenceBundle.js +47 -0
  136. package/dist/domain/hopperStartupFailure.js +1 -1
  137. package/dist/domain/hopperValues.js +14 -1
  138. package/dist/domain/inputIssueProjection.js +7 -1
  139. package/dist/domain/javascriptRuntimeObservation.js +170 -0
  140. package/dist/domain/javascriptRuntimeReconciliationParsing.js +120 -2
  141. package/dist/domain/javascriptRuntimeReconciliationResult.js +3 -3
  142. package/dist/domain/javascriptRuntimeReconciliationRuntime.js +2 -2
  143. package/dist/domain/javascriptRuntimeReconciliationSchemas.js +1 -1
  144. package/dist/domain/javascriptSemanticAnalysis.js +13 -16
  145. package/dist/domain/javascriptSemanticAsyncEffects.js +253 -0
  146. package/dist/domain/javascriptSemanticChildProcesses.js +237 -0
  147. package/dist/domain/javascriptSemanticDataEffectHelpers.js +125 -0
  148. package/dist/domain/javascriptSemanticDataEffects.js +284 -0
  149. package/dist/domain/javascriptSemanticDerivedAnalysis.js +41 -0
  150. package/dist/domain/javascriptSemanticFingerprints.js +174 -0
  151. package/dist/domain/javascriptSemanticIr.js +21 -1
  152. package/dist/domain/javascriptSemanticObjects.js +115 -0
  153. package/dist/domain/javascriptSemanticPromises.js +290 -0
  154. package/dist/domain/javascriptSemanticResources.js +136 -0
  155. package/dist/domain/javascriptSourceParser.js +15 -0
  156. package/dist/domain/javascriptStaticAnalysis.js +12 -15
  157. package/dist/domain/javascriptStaticAnalysisBundler.js +38 -0
  158. package/dist/domain/javascriptStaticAnalysisCalls.js +23 -1
  159. package/dist/domain/javascriptStaticAnalysisHelpers.js +8 -0
  160. package/dist/domain/mobileApplicationInvestigation.js +159 -0
  161. package/dist/domain/nativeApiBoundary.js +130 -0
  162. package/dist/domain/objcSwiftMetadata.js +150 -0
  163. package/dist/domain/packageAnalysis.js +123 -0
  164. package/dist/domain/peInspection.js +196 -0
  165. package/dist/domain/processCapture.js +8 -2
  166. package/dist/domain/processReactiveScenario.js +1 -0
  167. package/dist/domain/processReactiveTransition.js +9 -4
  168. package/dist/domain/protocolCapture.js +243 -0
  169. package/dist/domain/providerCleanupError.js +11 -0
  170. package/dist/domain/reconstructionObligationLedgerSchemas.js +278 -0
  171. package/dist/domain/reconstructionReadiness.js +121 -0
  172. package/dist/domain/reconstructionReadinessFindingHelpers.js +17 -0
  173. package/dist/domain/reconstructionReadinessIntegrityFindings.js +118 -0
  174. package/dist/domain/reconstructionReadinessJourneyFindings.js +193 -0
  175. package/dist/domain/reconstructionReadinessSchemas.js +240 -0
  176. package/dist/domain/sourceToBundleComparison.js +335 -0
  177. package/dist/domain/sourceToBundleComparisonSchemas.js +133 -0
  178. package/dist/domain/sourceToBundleSignals.js +202 -0
  179. package/dist/domain/webCaptureDiff.js +141 -27
  180. package/dist/domain/webScreenshot.js +1 -1
  181. package/dist/dotnet/ManagedStaticProvider.js +34 -5
  182. package/dist/generatedMcpToolCatalog.js +37 -0
  183. package/dist/generatedPackageMetadata.js +4 -4
  184. package/dist/ghidra/GhidraDefaults.js +1 -1
  185. package/dist/ghidra/GhidraFunctionValues.js +40 -2
  186. package/dist/ghidra/GhidraProviderCapabilities.js +47 -38
  187. package/dist/hopper/HopperCleanup.js +158 -0
  188. package/dist/hopper/HopperClient.js +93 -99
  189. package/dist/hopper/HopperProvider.js +61 -30
  190. package/dist/hopper/HopperRequestQueue.js +215 -0
  191. package/dist/hopper/HopperResponseStream.js +46 -0
  192. package/dist/hopper/protocol.js +63 -20
  193. package/dist/main/transport.js +24 -27
  194. package/dist/main.js +14 -6
  195. package/dist/mcpDoctor.js +27 -4
  196. package/dist/mcpStartupPolicy.js +14 -0
  197. package/dist/process/ProcessCommandIdentity.js +19 -0
  198. package/dist/process/ProcessOwnership.js +11 -8
  199. package/dist/server/DynamicToolAvailability.js +70 -0
  200. package/dist/server/LazyToolCatalog.js +134 -0
  201. package/dist/server/createServer.js +92 -154
  202. package/dist/server/hydrateServerTools.js +191 -0
  203. package/dist/server/lifecycleProgress.js +10 -0
  204. package/dist/server/permissionFailure.js +12 -0
  205. package/dist/server/registerApplicationTools/compareSourceToBundle.js +35 -0
  206. package/dist/server/registerApplicationTools/helpers.js +11 -7
  207. package/dist/server/registerApplicationTools/obligationLedger.js +27 -0
  208. package/dist/server/registerApplicationTools/readiness.js +55 -0
  209. package/dist/server/registerApplicationTools.js +6 -0
  210. package/dist/server/registerArtifactTools.js +9 -1
  211. package/dist/server/registerBrowserScenarioTool.js +25 -0
  212. package/dist/server/registerBrowserTools.js +2 -2
  213. package/dist/server/registerCloseLifecycleTool.js +57 -0
  214. package/dist/server/registerEnhancedTools.js +28 -7
  215. package/dist/server/registerEvidenceResources.js +54 -101
  216. package/dist/server/registerEvidenceTools.js +9 -2
  217. package/dist/server/registerInvestigationTools.js +1 -2
  218. package/dist/server/registerJavaScriptApplicationGraphResource.js +0 -4
  219. package/dist/server/registerJavaScriptRuntimeObservationTools.js +37 -0
  220. package/dist/server/registerReconstructionObligationLedgerResource.js +32 -0
  221. package/dist/server/registerReconstructionReadinessResource.js +32 -0
  222. package/dist/server/registerSessionRecordTools.js +82 -12
  223. package/dist/server/registerSessionStatusTool.js +11 -8
  224. package/dist/server/registerSessionTools.js +32 -58
  225. package/dist/server/sessionAvailabilityPolicy.js +10 -8
  226. package/dist/server/toolRegistrationOptions.js +21 -6
  227. package/dist/server/toolResult.js +12 -0
  228. package/package.json +28 -11
  229. package/scripts/package-runner-bootstrap.mjs +72 -0
  230. package/scripts/rea.mjs +11 -1
  231. package/skills/reverse-engineer-anything/SKILL.md +10 -3
@@ -0,0 +1,53 @@
1
+ import { createHash } from "node:crypto";
2
+ const REDACTION_PREFIX = "[REDACTED:";
3
+ /** Resolved secret values kept only for one in-memory scenario session. */
4
+ export class BrowserScenarioSecrets {
5
+ values;
6
+ constructor(values) {
7
+ this.values = values;
8
+ }
9
+ static resolve(scenario, environment) {
10
+ const values = new Map();
11
+ for (const declaration of scenario.secrets) {
12
+ const value = environment[declaration.environment_variable];
13
+ if (value === undefined)
14
+ return undefined;
15
+ values.set(declaration.secret_id, value);
16
+ }
17
+ return new BrowserScenarioSecrets(values);
18
+ }
19
+ value(source) {
20
+ if (source.source === "literal")
21
+ return source.value;
22
+ const value = this.values.get(source.secret_id);
23
+ if (value === undefined)
24
+ throw new Error("Validated browser secret was not resolved");
25
+ return value;
26
+ }
27
+ url(destination) {
28
+ const url = new URL(destination.url);
29
+ for (const { name, value } of destination.query)
30
+ url.searchParams.append(name, this.value(value));
31
+ return url.href;
32
+ }
33
+ redact(value) {
34
+ let output = value;
35
+ const replacements = [...this.values].sort(([leftId, left], [rightId, right]) => right.length - left.length ||
36
+ (leftId < rightId ? -1 : leftId > rightId ? 1 : 0));
37
+ for (const [id, secret] of replacements)
38
+ if (secret !== "")
39
+ output = output.replaceAll(secret, `${REDACTION_PREFIX}${id}]`);
40
+ return output;
41
+ }
42
+ fingerprint(value) {
43
+ for (const secret of this.values.values())
44
+ if (secret !== "" && value.includes(secret))
45
+ return { value_state: "redacted-secret", value_sha256: null };
46
+ return {
47
+ value_state: "hashed",
48
+ value_sha256: createHash("sha256")
49
+ .update(this.redact(value))
50
+ .digest("hex"),
51
+ };
52
+ }
53
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -3,7 +3,7 @@ import { analyzeCapturedWebBundle } from "../domain/webBundleAnalyzer.js";
3
3
  import { webBundleAnalysisSchema, } from "../domain/webBundleAnalysis.js";
4
4
  import { webObservationSessionSchema, } from "../domain/browserSession.js";
5
5
  import { webMcpDiscoverySchema, } from "../domain/webMcpDiscovery.js";
6
- import { compareWebCaptures, webCaptureDiffSchema, } from "../domain/webCaptureDiff.js";
6
+ import { browserCaptureComparisonSchema, compareBrowserCaptures, } from "../domain/browserCaptureComparison.js";
7
7
  import { webScreenshotDiffSchema, webScreenshotSchema, } from "../domain/webScreenshot.js";
8
8
  import { AnalysisError, BrowserObservationError, ProviderAdapterError, } from "../domain/errors.js";
9
9
  import { err, ok } from "../domain/result.js";
@@ -136,7 +136,7 @@ export class CdpBrowserProvider {
136
136
  }
137
137
  async compareCaptures(input) {
138
138
  try {
139
- return ok(webCaptureDiffSchema.parse(compareWebCaptures(input)));
139
+ return ok(browserCaptureComparisonSchema.parse(compareBrowserCaptures(input)));
140
140
  }
141
141
  catch (cause) {
142
142
  return err(providerError(cause, "compare_web_captures"));
@@ -156,6 +156,10 @@ export const captureDom = (result, allowedOrigins, input, completeness) => {
156
156
  };
157
157
  export const captureAccessibility = (results, maximum, options) => {
158
158
  const all = results.flatMap((result) => recordsValue(requiredRecord(result).nodes));
159
+ const nodeIds = new Set(all.flatMap((node) => {
160
+ const nodeId = stringValue(node.nodeId);
161
+ return nodeId === undefined ? [] : [nodeId];
162
+ }));
159
163
  let retainedBytes = 0;
160
164
  let excludedFields = 0;
161
165
  let truncatedFields = 0;
@@ -187,6 +191,7 @@ export const captureAccessibility = (results, maximum, options) => {
187
191
  name: captureText(node.name),
188
192
  description: captureText(node.description),
189
193
  ignored: node.ignored === true,
194
+ states: accessibilityStates(node.properties),
190
195
  };
191
196
  });
192
197
  for (const node of all.slice(maximum)) {
@@ -201,6 +206,11 @@ export const captureAccessibility = (results, maximum, options) => {
201
206
  return {
202
207
  total: all.length,
203
208
  nodes,
209
+ treeIncomplete: all.some((node) => Array.isArray(node.childIds) &&
210
+ node.childIds.some((childId) => {
211
+ const id = stringValue(childId);
212
+ return id !== undefined && !nodeIds.has(id);
213
+ })),
204
214
  textCapture: {
205
215
  status: options.unavailable
206
216
  ? "unavailable"
@@ -215,6 +225,53 @@ export const captureAccessibility = (results, maximum, options) => {
215
225
  },
216
226
  };
217
227
  };
228
+ const ACCESSIBILITY_STATE_NAMES = new Set([
229
+ "atomic",
230
+ "autocomplete",
231
+ "busy",
232
+ "checked",
233
+ "disabled",
234
+ "editable",
235
+ "expanded",
236
+ "focusable",
237
+ "focused",
238
+ "hasPopup",
239
+ "hidden",
240
+ "invalid",
241
+ "level",
242
+ "live",
243
+ "modal",
244
+ "multiline",
245
+ "multiselectable",
246
+ "orientation",
247
+ "pressed",
248
+ "readonly",
249
+ "required",
250
+ "selected",
251
+ "valuemax",
252
+ "valuemin",
253
+ ]);
254
+ const accessibilityStates = (value) => {
255
+ const states = new Map();
256
+ for (const property of recordsValue(value)) {
257
+ const name = stringValue(property.name);
258
+ if (name === undefined || !ACCESSIBILITY_STATE_NAMES.has(name))
259
+ continue;
260
+ const raw = recordValue(property.value)?.value;
261
+ const state = typeof raw === "boolean"
262
+ ? raw
263
+ : typeof raw === "number"
264
+ ? numberValue(raw)
265
+ : typeof raw === "string"
266
+ ? raw.slice(0, 256)
267
+ : undefined;
268
+ if (state !== undefined)
269
+ states.set(name, state);
270
+ }
271
+ return [...states]
272
+ .sort(([left], [right]) => left.localeCompare(right))
273
+ .map(([name, state]) => ({ name, value: state }));
274
+ };
218
275
  const axText = (value) => boundedText(recordValue(value)?.value, 1_024);
219
276
  const indexedString = (strings, index) => {
220
277
  const integer = numberValue(index);
@@ -1,53 +1,95 @@
1
1
  import { CdpConnection } from "./CdpConnection.js";
2
2
  import { optionalCdpCommand } from "./CdpOptionalCommand.js";
3
+ import { captureStorageFingerprints, } from "./CdpCaptureStorageFingerprints.js";
3
4
  import { numberValue, recordValue, recordsValue, stringValue, } from "./CdpCaptureValues.js";
4
5
  /** Capture redacted, bounded storage metadata for one authorized origin. */
5
6
  export const captureStorage = async (context, origin, limitations) => {
6
7
  const quota = recordValue(await optionalCdpCommand(context, "Storage.getUsageAndQuota", { origin }, limitations));
7
8
  const local = context.input.include_storage_keys
8
- ? await storageKeys(context, origin, true, limitations)
9
- : [];
9
+ ? await storageItems(context, origin, true, limitations)
10
+ : emptyStorageItems;
10
11
  const session = context.input.include_storage_keys
11
- ? await storageKeys(context, origin, false, limitations)
12
- : [];
13
- const indexed = context.input.include_storage_keys
14
- ? stringArray(recordValue(await optionalCdpCommand(context, "IndexedDB.requestDatabaseNames", { securityOrigin: origin }, limitations))?.databaseNames)
15
- : [];
16
- const caches = context.input.include_storage_keys
17
- ? recordsValue(recordValue(await optionalCdpCommand(context, "CacheStorage.requestCacheNames", { securityOrigin: origin }, limitations))?.caches).flatMap((cache) => {
18
- const name = stringValue(cache.cacheName);
19
- return name === undefined ? [] : [name.slice(0, 1_024)];
20
- })
21
- : [];
12
+ ? await storageItems(context, origin, false, limitations)
13
+ : emptyStorageItems;
14
+ const indexedRaw = context.input.include_storage_keys
15
+ ? await optionalCdpCommand(context, "IndexedDB.requestDatabaseNames", { securityOrigin: origin }, limitations)
16
+ : undefined;
17
+ const indexedValues = recordValue(indexedRaw)?.databaseNames;
18
+ const indexed = stringArray(indexedValues);
19
+ const cacheRaw = context.input.include_storage_keys
20
+ ? await optionalCdpCommand(context, "CacheStorage.requestCacheNames", { securityOrigin: origin }, limitations)
21
+ : undefined;
22
+ const cacheValues = recordsValue(recordValue(cacheRaw)?.caches);
23
+ const caches = capturedCaches(cacheValues);
22
24
  const maximum = context.input.limits.max_storage_keys;
25
+ const fingerprints = context.input.include_storage_fingerprints
26
+ ? await captureStorageFingerprints({
27
+ context,
28
+ origin,
29
+ local,
30
+ session,
31
+ indexedDbNames: indexed.slice(0, maximum),
32
+ caches: caches.slice(0, maximum),
33
+ limitations,
34
+ })
35
+ : { items: [], complete: false, truncated: false };
36
+ const truncated = [local.items, session.items, indexed, caches].some((items) => items.length > maximum) || fingerprints.truncated;
37
+ const structuredStorageComplete = indexedRaw !== undefined &&
38
+ Array.isArray(indexedValues) &&
39
+ indexed.length === indexedValues.length &&
40
+ cacheRaw !== undefined &&
41
+ caches.length === cacheValues.length;
23
42
  return {
24
43
  value: {
25
44
  origin,
26
45
  usage_bytes: numberValue(quota?.usage) ?? null,
27
46
  quota_bytes: numberValue(quota?.quota) ?? null,
28
- local_storage_keys: local.slice(0, maximum),
29
- session_storage_keys: session.slice(0, maximum),
30
- indexed_db_names: indexed.slice(0, maximum),
31
- cache_names: caches.slice(0, maximum),
47
+ local_storage_keys: local.items
48
+ .map(({ key }) => key.slice(0, 1_024))
49
+ .slice(0, maximum),
50
+ session_storage_keys: session.items
51
+ .map(({ key }) => key.slice(0, 1_024))
52
+ .slice(0, maximum),
53
+ indexed_db_names: indexed
54
+ .map((name) => name.slice(0, 1_024))
55
+ .slice(0, maximum),
56
+ cache_names: caches
57
+ .map(({ name }) => name.slice(0, 1_024))
58
+ .slice(0, maximum),
59
+ content_fingerprints: [...fingerprints.items],
60
+ fingerprint_algorithm: "sha256",
61
+ fingerprints_complete: fingerprints.complete && structuredStorageComplete && !truncated,
32
62
  values_redacted: true,
33
63
  },
34
- truncated: [local, session, indexed, caches].some((items) => items.length > maximum),
64
+ truncated,
35
65
  };
36
66
  };
37
- const storageKeys = async (context, origin, isLocalStorage, limitations) => {
38
- const result = recordValue(await optionalCdpCommand(context, "DOMStorage.getDOMStorageItems", { storageId: { securityOrigin: origin, isLocalStorage } }, limitations));
39
- return Array.isArray(result?.entries)
40
- ? result.entries.flatMap((entry) => {
41
- if (!Array.isArray(entry))
42
- return [];
43
- const key = stringValue(entry[0]);
44
- return key === undefined ? [] : [key.slice(0, 1_024)];
45
- })
46
- : [];
67
+ const emptyStorageItems = {
68
+ items: [],
69
+ complete: false,
70
+ };
71
+ const capturedCaches = (values) => values.flatMap((cache) => {
72
+ const name = stringValue(cache.cacheName);
73
+ const id = stringValue(cache.cacheId);
74
+ return name === undefined || id === undefined ? [] : [{ name, id }];
75
+ });
76
+ const storageItems = async (context, origin, isLocalStorage, limitations) => {
77
+ const raw = await optionalCdpCommand(context, "DOMStorage.getDOMStorageItems", { storageId: { securityOrigin: origin, isLocalStorage } }, limitations);
78
+ const entries = recordValue(raw)?.entries;
79
+ if (raw === undefined || !Array.isArray(entries))
80
+ return { items: [], complete: false };
81
+ const items = entries.flatMap((entry) => {
82
+ if (!Array.isArray(entry))
83
+ return [];
84
+ const key = stringValue(entry[0]);
85
+ const value = stringValue(entry[1]);
86
+ return key === undefined || value === undefined ? [] : [{ key, value }];
87
+ });
88
+ return { items, complete: items.length === entries.length };
47
89
  };
48
90
  const stringArray = (value) => Array.isArray(value)
49
91
  ? value.flatMap((item) => {
50
92
  const text = stringValue(item);
51
- return text === undefined ? [] : [text.slice(0, 1_024)];
93
+ return text === undefined ? [] : [text];
52
94
  })
53
95
  : [];
@@ -0,0 +1,328 @@
1
+ import { createHash } from "node:crypto";
2
+ import canonicalize from "canonicalize";
3
+ import { CdpConnection } from "./CdpConnection.js";
4
+ import { optionalCdpCommand } from "./CdpOptionalCommand.js";
5
+ import { recordValue, recordsValue, stringValue } from "./CdpCaptureValues.js";
6
+ const MAX_CACHE_BODY_BYTES = 64 * 1_024;
7
+ /** Capture stable hashes of approved storage content without returning values. */
8
+ export const captureStorageFingerprints = async ({ context, origin, local, session, indexedDbNames, caches, limitations, }) => {
9
+ const state = {
10
+ items: [],
11
+ maximum: context.input.limits.max_storage_keys,
12
+ complete: local.complete && session.complete,
13
+ truncated: false,
14
+ };
15
+ const run = { context, limitations, state };
16
+ addStorageItems(state, "local_storage", local.items);
17
+ addStorageItems(state, "session_storage", session.items);
18
+ await addCookies(run, origin);
19
+ await addIndexedDb(run, origin, indexedDbNames);
20
+ await addCaches(run, caches);
21
+ state.items.sort((left, right) => compareCodePoints(`${left.scope}:${left.identity_sha256}`, `${right.scope}:${right.identity_sha256}`));
22
+ return {
23
+ items: state.items,
24
+ complete: state.complete &&
25
+ !state.truncated &&
26
+ state.items.every(({ complete }) => complete),
27
+ truncated: state.truncated,
28
+ };
29
+ };
30
+ const addStorageItems = (state, scope, items) => {
31
+ for (const item of items)
32
+ addFingerprint(state, {
33
+ scope,
34
+ identity: item.key,
35
+ value: item.value,
36
+ complete: true,
37
+ });
38
+ };
39
+ const addCookies = async (run, origin) => {
40
+ const raw = await optionalCdpCommand(run.context, "Network.getCookies", { urls: [origin] }, run.limitations);
41
+ if (raw === undefined) {
42
+ run.state.complete = false;
43
+ return;
44
+ }
45
+ const cookieValues = recordValue(raw)?.cookies;
46
+ if (!Array.isArray(cookieValues)) {
47
+ run.state.complete = false;
48
+ return;
49
+ }
50
+ const cookies = recordsValue(cookieValues);
51
+ if (cookies.length !== cookieValues.length)
52
+ run.state.complete = false;
53
+ for (const cookie of cookies) {
54
+ const name = stringValue(cookie.name);
55
+ const value = stringValue(cookie.value);
56
+ const domain = stringValue(cookie.domain);
57
+ const path = stringValue(cookie.path);
58
+ if (name === undefined ||
59
+ value === undefined ||
60
+ domain === undefined ||
61
+ path === undefined) {
62
+ run.state.complete = false;
63
+ continue;
64
+ }
65
+ addFingerprint(run.state, {
66
+ scope: "cookie",
67
+ identity: {
68
+ domain,
69
+ path,
70
+ name,
71
+ partition_key: cookie.partitionKey ?? null,
72
+ },
73
+ value,
74
+ complete: true,
75
+ });
76
+ }
77
+ };
78
+ const addIndexedDb = async (run, origin, names) => {
79
+ for (const databaseName of names) {
80
+ if (run.state.items.length >= run.state.maximum) {
81
+ run.state.truncated = true;
82
+ return;
83
+ }
84
+ const raw = await optionalCdpCommand(run.context, "IndexedDB.requestDatabase", { securityOrigin: origin, databaseName }, run.limitations);
85
+ const database = recordValue(recordValue(raw)?.databaseWithObjectStores);
86
+ if (raw === undefined || database === undefined) {
87
+ run.state.complete = false;
88
+ continue;
89
+ }
90
+ if (stringValue(database.name) !== databaseName) {
91
+ run.state.complete = false;
92
+ continue;
93
+ }
94
+ if (!Array.isArray(database.objectStores)) {
95
+ run.state.complete = false;
96
+ continue;
97
+ }
98
+ const stores = recordsValue(database.objectStores);
99
+ if (stores.length !== database.objectStores.length)
100
+ run.state.complete = false;
101
+ addFingerprint(run.state, {
102
+ scope: "indexed_db_schema",
103
+ identity: databaseName,
104
+ value: stores.map(indexedDbStoreSchema),
105
+ complete: indexedDbSchemaComplete(stores),
106
+ });
107
+ for (const store of stores) {
108
+ const objectStoreName = stringValue(store.name);
109
+ if (objectStoreName === undefined) {
110
+ run.state.complete = false;
111
+ continue;
112
+ }
113
+ await addIndexedDbRecords(run, {
114
+ origin,
115
+ databaseName,
116
+ objectStoreName,
117
+ });
118
+ }
119
+ }
120
+ };
121
+ const indexedDbStoreSchema = (store) => ({
122
+ name: stringValue(store.name) ?? null,
123
+ key_path: stableValue(store.keyPath),
124
+ auto_increment: store.autoIncrement === true,
125
+ indexes: recordsValue(store.indexes).map((index) => ({
126
+ name: stringValue(index.name) ?? null,
127
+ key_path: stableValue(index.keyPath),
128
+ unique: index.unique === true,
129
+ multi_entry: index.multiEntry === true,
130
+ })),
131
+ });
132
+ const indexedDbSchemaComplete = (stores) => stores.every((store) => stringValue(store.name) !== undefined &&
133
+ stableValueComplete(store.keyPath) &&
134
+ typeof store.autoIncrement === "boolean" &&
135
+ Array.isArray(store.indexes) &&
136
+ recordsValue(store.indexes).length === store.indexes.length &&
137
+ recordsValue(store.indexes).every((index) => stringValue(index.name) !== undefined &&
138
+ stableValueComplete(index.keyPath) &&
139
+ typeof index.unique === "boolean" &&
140
+ typeof index.multiEntry === "boolean"));
141
+ const addIndexedDbRecords = async (run, input) => {
142
+ const remaining = run.state.maximum - run.state.items.length;
143
+ if (remaining <= 0) {
144
+ run.state.truncated = true;
145
+ return;
146
+ }
147
+ const raw = await optionalCdpCommand(run.context, "IndexedDB.requestData", {
148
+ securityOrigin: input.origin,
149
+ databaseName: input.databaseName,
150
+ objectStoreName: input.objectStoreName,
151
+ indexName: "",
152
+ skipCount: 0,
153
+ pageSize: remaining,
154
+ }, run.limitations);
155
+ const result = recordValue(raw);
156
+ if (raw === undefined || result === undefined) {
157
+ run.state.complete = false;
158
+ return;
159
+ }
160
+ if (!Array.isArray(result.objectStoreDataEntries) ||
161
+ typeof result.hasMore !== "boolean") {
162
+ run.state.complete = false;
163
+ return;
164
+ }
165
+ const entries = recordsValue(result.objectStoreDataEntries);
166
+ if (entries.length !== result.objectStoreDataEntries.length)
167
+ run.state.complete = false;
168
+ for (const entry of entries) {
169
+ const key = stableValue(entry.key);
170
+ const primaryKey = stableValue(entry.primaryKey);
171
+ const value = stableValue(entry.value);
172
+ addFingerprint(run.state, {
173
+ scope: "indexed_db_record",
174
+ identity: {
175
+ database: input.databaseName,
176
+ store: input.objectStoreName,
177
+ key,
178
+ primaryKey,
179
+ },
180
+ value,
181
+ complete: remoteObjectComplete(entry.value),
182
+ });
183
+ }
184
+ if (result.hasMore === true || entries.length > remaining)
185
+ run.state.truncated = true;
186
+ };
187
+ const addCaches = async (run, caches) => {
188
+ for (const cache of caches) {
189
+ const remaining = run.state.maximum - run.state.items.length;
190
+ if (remaining <= 0) {
191
+ run.state.truncated = true;
192
+ return;
193
+ }
194
+ const raw = await optionalCdpCommand(run.context, "CacheStorage.requestEntries", { cacheId: cache.id, skipCount: 0, pageSize: remaining }, run.limitations);
195
+ const result = recordValue(raw);
196
+ if (raw === undefined || result === undefined) {
197
+ run.state.complete = false;
198
+ continue;
199
+ }
200
+ if (!Array.isArray(result.cacheDataEntries) ||
201
+ typeof result.returnCount !== "number") {
202
+ run.state.complete = false;
203
+ continue;
204
+ }
205
+ const entries = recordsValue(result.cacheDataEntries);
206
+ if (entries.length !== result.cacheDataEntries.length)
207
+ run.state.complete = false;
208
+ for (const entry of entries)
209
+ await addCacheEntry(run, cache, entry);
210
+ if (result.returnCount > entries.length)
211
+ run.state.truncated = true;
212
+ }
213
+ };
214
+ const addCacheEntry = async (run, cache, entry) => {
215
+ const requestURL = stringValue(entry.requestURL);
216
+ const requestMethod = stringValue(entry.requestMethod);
217
+ if (requestURL === undefined || requestMethod === undefined) {
218
+ run.state.complete = false;
219
+ return;
220
+ }
221
+ if (!Array.isArray(entry.requestHeaders)) {
222
+ run.state.complete = false;
223
+ return;
224
+ }
225
+ const headerRecords = recordsValue(entry.requestHeaders);
226
+ const requestHeaders = headerRecords.flatMap((header) => {
227
+ const name = stringValue(header.name);
228
+ const value = stringValue(header.value);
229
+ return name === undefined || value === undefined ? [] : [{ name, value }];
230
+ });
231
+ if (headerRecords.length !== entry.requestHeaders.length ||
232
+ requestHeaders.length !== headerRecords.length) {
233
+ run.state.complete = false;
234
+ return;
235
+ }
236
+ const raw = await optionalCdpCommand(run.context, "CacheStorage.requestCachedResponse", { cacheId: cache.id, requestURL, requestHeaders }, run.limitations);
237
+ const body = stringValue(recordValue(recordValue(raw)?.response)?.body);
238
+ const decoded = body === undefined ? null : decodeBase64(body);
239
+ const complete = decoded !== null && decoded.byteLength <= MAX_CACHE_BODY_BYTES;
240
+ if (!complete)
241
+ run.state.complete = false;
242
+ addFingerprint(run.state, {
243
+ scope: "cache_entry",
244
+ identity: { cache: cache.name, requestURL, requestMethod, requestHeaders },
245
+ value: decoded ?? undefined,
246
+ complete,
247
+ });
248
+ };
249
+ const addFingerprint = (state, input) => {
250
+ if (state.items.length >= state.maximum) {
251
+ state.truncated = true;
252
+ return;
253
+ }
254
+ state.items.push({
255
+ scope: input.scope,
256
+ identity_sha256: digest(input.identity),
257
+ value_sha256: input.value === undefined ? null : digest(input.value),
258
+ complete: input.complete &&
259
+ stableValueComplete(input.identity) &&
260
+ stableValueComplete(input.value),
261
+ });
262
+ };
263
+ const digest = (value) => createHash("sha256").update(encodedValue(value)).digest("hex");
264
+ const encodedValue = (value) => {
265
+ if (value instanceof Uint8Array)
266
+ return value;
267
+ const encoded = canonicalize(stableValue(value));
268
+ if (encoded === undefined)
269
+ throw new TypeError("Expected canonical JSON");
270
+ return encoded;
271
+ };
272
+ const stableValue = (value, depth = 0) => {
273
+ if (value === null ||
274
+ typeof value === "string" ||
275
+ typeof value === "number" ||
276
+ typeof value === "boolean")
277
+ return value;
278
+ if (depth >= 8)
279
+ return "<depth-limit>";
280
+ if (Array.isArray(value))
281
+ return value.slice(0, 64).map((item) => stableValue(item, depth + 1));
282
+ const record = recordValue(value);
283
+ if (record === undefined)
284
+ return String(value);
285
+ return Object.fromEntries(Object.entries(record)
286
+ .filter(([key]) => key !== "objectId" && key !== "customPreview")
287
+ .sort(([left], [right]) => compareCodePoints(left, right))
288
+ .slice(0, 64)
289
+ .map(([key, item]) => [key, stableValue(item, depth + 1)]));
290
+ };
291
+ const stableValueComplete = (value, depth = 0) => {
292
+ if (value instanceof Uint8Array)
293
+ return true;
294
+ if (value === null ||
295
+ value === undefined ||
296
+ typeof value === "string" ||
297
+ typeof value === "number" ||
298
+ typeof value === "boolean")
299
+ return true;
300
+ if (depth >= 8)
301
+ return false;
302
+ if (Array.isArray(value))
303
+ return (value.length <= 64 &&
304
+ value.every((item) => stableValueComplete(item, depth + 1)));
305
+ const record = recordValue(value);
306
+ if (record === undefined)
307
+ return false;
308
+ const entries = Object.entries(record).filter(([key]) => key !== "objectId" && key !== "customPreview");
309
+ return (entries.length <= 64 &&
310
+ entries.every(([, item]) => stableValueComplete(item, depth + 1)));
311
+ };
312
+ const remoteObjectComplete = (value) => {
313
+ const record = recordValue(value);
314
+ if (record === undefined)
315
+ return true;
316
+ return (record.objectId === undefined &&
317
+ record.preview === undefined &&
318
+ record.customPreview === undefined &&
319
+ (record.type !== "object" || record.value !== undefined) &&
320
+ stableValueComplete(record));
321
+ };
322
+ const decodeBase64 = (value) => {
323
+ if (value.length % 4 !== 0 ||
324
+ !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(value))
325
+ return null;
326
+ return Buffer.from(value, "base64");
327
+ };
328
+ const compareCodePoints = (left, right) => left < right ? -1 : left > right ? 1 : 0;
@@ -24,10 +24,10 @@ const endpointTargetSchema = z.object({
24
24
  const endpointTargetsSchema = z.array(endpointTargetSchema).max(1_000);
25
25
  /** Read bounded CDP discovery endpoints without following redirects. */
26
26
  export const discoverCdpEndpoint = async (endpoint, operation, signal) => {
27
- const versionInput = await readJson(new URL("/json/version", endpoint), MAX_VERSION_BYTES, operation, signal);
28
- const targetsInput = await readJson(new URL("/json/list", endpoint), MAX_TARGET_LIST_BYTES, operation, signal);
29
- const version = parseEndpointValue(endpointVersionSchema, versionInput, operation);
30
- const targets = parseEndpointValue(endpointTargetsSchema, targetsInput, operation);
27
+ const versionInput = await readBoundedCdpJson(new URL("/json/version", endpoint), MAX_VERSION_BYTES, operation, signal);
28
+ const targetsInput = await readBoundedCdpJson(new URL("/json/list", endpoint), MAX_TARGET_LIST_BYTES, operation, signal);
29
+ const version = parseCdpEndpointValue(endpointVersionSchema, versionInput, operation);
30
+ const targets = parseCdpEndpointValue(endpointTargetsSchema, targetsInput, operation);
31
31
  return {
32
32
  webSocket: safeCdpWebSocketEndpoint(endpoint, version.webSocketDebuggerUrl, operation),
33
33
  version: {
@@ -110,7 +110,8 @@ const availableCdpTargetWebSocket = (discovery, target) => {
110
110
  ? discovery.webSocket
111
111
  : undefined;
112
112
  };
113
- const parseEndpointValue = (schema, input, operation) => {
113
+ /** Parse one CDP discovery value through its exact boundary schema. */
114
+ export const parseCdpEndpointValue = (schema, input, operation) => {
114
115
  const parsed = schema.safeParse(input);
115
116
  if (!parsed.success)
116
117
  throw new BrowserObservationError(operation, "invalid_endpoint_response", {
@@ -173,7 +174,8 @@ const targetWebSocket = (endpoint, target, operation) => {
173
174
  return {};
174
175
  }
175
176
  };
176
- const readJson = async (url, maximumBytes, operation, signal) => await new Promise((resolve, reject) => {
177
+ /** Read one bounded loopback CDP discovery document without redirects. */
178
+ export const readBoundedCdpJson = async (url, maximumBytes, operation, signal) => await new Promise((resolve, reject) => {
177
179
  if (signal?.aborted === true) {
178
180
  reject(new AnalysisCancelledError(operation));
179
181
  return;
@@ -226,6 +226,8 @@ const accessibilityForFrames = async (state, frames) => {
226
226
  events.completeness.exclude("accessibility", "not_approved", capture.textCapture.excluded_fields);
227
227
  if (capture.textCapture.status === "truncated")
228
228
  events.completeness.truncate("accessibility");
229
+ if (capture.treeIncomplete)
230
+ events.completeness.truncate("accessibility");
229
231
  return capture;
230
232
  };
231
233
  const publicResource = ({ rawUrl: _rawUrl, ...resource }) => resource;