rea-agents 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -12
- package/bridge/hopper_bridge.py +12 -3
- package/dist/application/AnalysisSnapshotCache.js +8 -6
- package/dist/application/ArtifactGraphConstruction.js +12 -3
- package/dist/application/ArtifactInventory.js +100 -27
- package/dist/application/AuthorizedArtifactInventory.js +4 -3
- package/dist/application/BinarySession.js +92 -4
- package/dist/application/BoundedJsonFiles.js +6 -29
- package/dist/application/BrowserEvidence.js +35 -0
- package/dist/application/BrowserObservationPort.js +1 -0
- package/dist/application/BrowserObservationService.js +52 -0
- package/dist/application/CapabilityInventory.js +134 -0
- package/dist/application/ClientRegistrationStatus.js +69 -0
- package/dist/application/ConfiguredRoots.js +19 -0
- package/dist/application/CrossVersionInventory.js +19 -8
- package/dist/application/CrossVersionInvestigation.js +74 -5
- package/dist/application/DeferredFileAuthorization.js +61 -0
- package/dist/application/DirectAnalysis.js +117 -9
- package/dist/application/Doctor.js +74 -2
- package/dist/application/EvidenceLedger.js +6 -5
- package/dist/application/InvestigationWorkspaceStore.js +2 -26
- package/dist/application/PermissionAuthority.js +183 -0
- package/dist/application/PermissionConfiguration.js +26 -0
- package/dist/application/ProcessCaptureAuthority.js +3 -2
- package/dist/application/ProcessCaptureError.js +13 -0
- package/dist/application/ProcessCaptureLifecycle.js +21 -9
- package/dist/application/ProcessCheckpoints.js +1 -1
- package/dist/application/ProcessCli.js +40 -15
- package/dist/application/ProcessHarness.js +10 -20
- package/dist/application/ProcessNormalization.js +17 -2
- package/dist/application/ProcessOwnership.js +4 -3
- package/dist/application/ProcessSampling.js +52 -17
- package/dist/application/ProgressReporter.js +30 -0
- package/dist/application/ProjectPermissionStore.js +213 -0
- package/dist/application/ReferenceSourceImportPolicy.js +3 -2
- package/dist/application/Setup.js +2 -41
- package/dist/application/SupportedClients.js +41 -0
- package/dist/application/TerminalRenderer.js +1 -1
- package/dist/application/Uninstall.js +1 -1
- package/dist/application/Upgrade.js +2 -0
- package/dist/application/runtime.js +1 -1
- package/dist/artifacts/ArtifactProvider.js +13 -3
- package/dist/artifacts/MachOSliceArtifactReader.js +1 -1
- package/dist/artifacts/NativeDmgArtifactReader.js +1 -1
- package/dist/artifacts/SafeOutputTree.js +3 -3
- package/dist/browser/CdpBrowserProvider.js +158 -0
- package/dist/browser/CdpCaptureDocuments.js +140 -0
- package/dist/browser/CdpCaptureEvents.js +248 -0
- package/dist/browser/CdpCaptureStorage.js +53 -0
- package/dist/browser/CdpCaptureValues.js +70 -0
- package/dist/browser/CdpConnection.js +218 -0
- package/dist/browser/CdpEndpoint.js +114 -0
- package/dist/browser/CdpOptionalCommand.js +15 -0
- package/dist/browser/CdpPageCapture.js +273 -0
- package/dist/catalogIdentity.js +123 -0
- package/dist/cli.js +48 -3
- package/dist/cliBrowserCommands.js +136 -0
- package/dist/cliEvidenceCommands.js +42 -1
- package/dist/cliInvestigationCommands.js +42 -6
- package/dist/cliLogging.js +20 -3
- package/dist/cliPolicyCommands.js +118 -0
- package/dist/cliProcessCommands.js +4 -14
- package/dist/config.js +107 -3
- package/dist/contracts/artifactToolContracts.js +14 -1
- package/dist/contracts/browserToolContracts.js +80 -0
- package/dist/contracts/errorSchemas.js +100 -0
- package/dist/contracts/toolContracts.js +13 -6
- package/dist/contracts/toolOutputSchemas.js +62 -30
- package/dist/domain/artifactComparison.js +38 -11
- package/dist/domain/artifactGraph.js +25 -1
- package/dist/domain/artifactInventoryEvidence.js +18 -0
- package/dist/domain/browserObservation.js +361 -0
- package/dist/domain/changedBehavior.js +9 -18
- package/dist/domain/errors.js +257 -3
- package/dist/domain/investigationWorkspace.js +131 -4
- package/dist/domain/jsonLimits.js +26 -0
- package/dist/domain/permissionPolicy.js +161 -0
- package/dist/domain/processCapture.js +3 -1
- package/dist/domain/processComparison.js +23 -34
- package/dist/domain/reconstructionVerification.js +3 -16
- package/dist/domain/reconstructionVerificationSchemas.js +3 -0
- package/dist/domain/staticRuntimeCorrelation.js +7 -1
- package/dist/domain/symbolAnalysis.js +1 -1
- package/dist/generatedPackageMetadata.js +9 -0
- package/dist/hopper/BridgeLauncher.js +7 -3
- package/dist/hopper/HopperClient.js +32 -5
- package/dist/identity.js +10 -3
- package/dist/logger.js +2 -2
- package/dist/main.js +101 -7
- package/dist/native/CommandRunner.js +11 -4
- package/dist/server/createServer.js +59 -5
- package/dist/server/mcpProgress.js +23 -0
- package/dist/server/registerArtifactComparisonTool.js +6 -2
- package/dist/server/registerBrowserTools.js +36 -0
- package/dist/server/registerBundleComparisonTool.js +6 -2
- package/dist/server/registerEnhancedTools.js +18 -1
- package/dist/server/registerEvidenceResources.js +302 -0
- package/dist/server/registerEvidenceTools.js +58 -1
- package/dist/server/registerFunctionComparisonTool.js +6 -2
- package/dist/server/registerInvestigationTools.js +78 -10
- package/dist/server/registerOfficialTools.js +19 -1
- package/dist/server/registerProcessComparisonTool.js +9 -3
- package/dist/server/registerSessionStatusTool.js +44 -3
- package/dist/server/registerSessionTools.js +144 -9
- package/dist/server/runDerivedOperation.js +43 -0
- package/dist/server/toolResult.js +27 -3
- package/dist/serverIdentity.js +70 -0
- package/package.json +7 -4
- package/scripts/rea.mjs +1 -1
- package/skills/rea-analysis/SKILL.md +18 -3
package/README.md
CHANGED
|
@@ -10,16 +10,20 @@
|
|
|
10
10
|
|
|
11
11
|
[](https://www.npmjs.com/package/rea-agents)
|
|
12
12
|
[](https://github.com/morluto/rea/actions/workflows/ci.yml)
|
|
13
|
-
[](#70-tools-for-investigation)
|
|
14
14
|
[](https://nodejs.org/)
|
|
15
15
|
[](LICENSE)
|
|
16
16
|
|
|
17
|
-
[Quick start](#quick-start) · [Current status](#current-status) · [Investigation model](#the-investigation-model) · [
|
|
17
|
+
[Quick start](#quick-start) · [Current status](#current-status) · [Investigation model](#the-investigation-model) · [70 tools](#70-tools-for-investigation) · [Roadmap](#roadmap) · [How it works](#how-it-works)
|
|
18
18
|
|
|
19
19
|
<br />
|
|
20
20
|
|
|
21
21
|
<code>npm install --global rea-agents && rea setup</code>
|
|
22
22
|
|
|
23
|
+
<br />
|
|
24
|
+
|
|
25
|
+
<img src="docs/assets/rea-hopper-analysis.png" alt="REA launching its analysis bridge inside Hopper while inspecting a native binary" width="1200" />
|
|
26
|
+
|
|
23
27
|
</div>
|
|
24
28
|
|
|
25
29
|
---
|
|
@@ -290,15 +294,16 @@ REA handles the app analysis in steps 1–5. The agent performs step 6 with its
|
|
|
290
294
|
- Analyze Swift and Objective-C metadata without manually untangling every mangled symbol.
|
|
291
295
|
- Leave names, comments, and bookmarks in Hopper so human and agent analysis reinforce each other.
|
|
292
296
|
|
|
293
|
-
##
|
|
297
|
+
## 70 tools for investigation
|
|
294
298
|
|
|
295
|
-
| Tool family | Count | Examples
|
|
296
|
-
| ------------------------- | ----: |
|
|
297
|
-
| Native inspection | 33 | procedures, pseudocode, assembly, strings, names, segments, callers, callees, xrefs, annotations
|
|
298
|
-
| Investigation workflows | 10 | `binary_overview`, `analyze_function`, `batch_decompile`, `trace_feature`, call graphs, Swift and Objective-C discovery
|
|
299
|
-
| Native macOS utilities | 5 | Mach-O metadata, code signatures, plists, architectures, Swift demangling; Hopper-free and provenance-bearing
|
|
300
|
-
| Artifact graph | 2 | deterministic directory, ZIP/APK/IPA, and ASAR inventory; explicitly selected extraction into an absent owned tree
|
|
301
|
-
|
|
|
299
|
+
| Tool family | Count | Examples |
|
|
300
|
+
| ------------------------- | ----: | ---------------------------------------------------------------------------------------------------------------------------------- |
|
|
301
|
+
| Native inspection | 33 | procedures, pseudocode, assembly, strings, names, segments, callers, callees, xrefs, annotations |
|
|
302
|
+
| Investigation workflows | 10 | `binary_overview`, `analyze_function`, `batch_decompile`, `trace_feature`, call graphs, Swift and Objective-C discovery |
|
|
303
|
+
| Native macOS utilities | 5 | Mach-O metadata, code signatures, plists, architectures, Swift demangling; Hopper-free and provenance-bearing |
|
|
304
|
+
| Artifact graph | 2 | deterministic directory, ZIP/APK/IPA, and ASAR inventory; explicitly selected extraction into an absent owned tree |
|
|
305
|
+
| Browser observation | 2 | exact-origin CDP page discovery and passive DOM, accessibility, script, resource, network, console, worker, and storage inspection |
|
|
306
|
+
| Workspace and observation | 18 | target lifecycle, Evidence v2 bundles, process/artifact/function comparison, evidence-linked residual-unknown lifecycle |
|
|
302
307
|
|
|
303
308
|
The public interface describes what the agent is trying to learn. Providers decide how to answer. macOS utilities handle common semantic inspection without launching Hopper; Hopper handles deeper native analysis; the process harness implements controlled behavioral capture.
|
|
304
309
|
|
|
@@ -307,6 +312,7 @@ The public interface describes what the agent is trying to learn. Providers deci
|
|
|
307
312
|
REA is already useful for native application investigation on macOS:
|
|
308
313
|
|
|
309
314
|
- Open Mach-O, ELF, PE, `.app`, ZIP, APK, IPA, ASAR, plist, JavaScript, source-map, and Hopper database targets.
|
|
315
|
+
- Attach to a user-owned Chrome-family browser over a configured loopback CDP endpoint, list only pages on approved exact origins, and capture bounded passive web evidence without navigation or JavaScript evaluation.
|
|
310
316
|
- Traverse content-addressed artifact graphs without extraction; on macOS, read-only DMG traversal additionally requires `native_mount_approved: true` and `REA_ARTIFACT_NATIVE_MOUNT_ENABLED=true`. Materialize only approved occurrences into absent output roots.
|
|
311
317
|
- Build bounded function dossiers with pseudocode, assembly, CFG edges, comments, calls, references, strings, and names.
|
|
312
318
|
- Search and trace features across symbols, strings, metadata, references, and call paths.
|
|
@@ -327,12 +333,27 @@ REA is already useful for native application investigation on macOS:
|
|
|
327
333
|
|
|
328
334
|
Hopper is the first provider, not the boundary of the project. Some current workflows still require Hopper and macOS; every evidence record identifies the provider and limitations behind its result.
|
|
329
335
|
|
|
336
|
+
### Website observation with CDP
|
|
337
|
+
|
|
338
|
+
REA can inspect an already-running Chrome-family browser that you own. Browser observation is disabled by default and requires a literal loopback CDP endpoint plus exact approved page origins:
|
|
339
|
+
|
|
340
|
+
```bash
|
|
341
|
+
export REA_BROWSER_OBSERVE_ENABLED=true
|
|
342
|
+
export REA_BROWSER_CDP_ENDPOINTS_JSON='["http://127.0.0.1:9222"]'
|
|
343
|
+
export REA_BROWSER_ALLOWED_ORIGINS_JSON='["http://127.0.0.1:3000"]'
|
|
344
|
+
|
|
345
|
+
rea list-browser-targets http://127.0.0.1:9222 --approved --json
|
|
346
|
+
rea inspect-web-page http://127.0.0.1:9222 TARGET_ID --approved --json
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
`list_browser_targets` and `inspect_web_page` expose the same Evidence v2 contracts over MCP. Inspection is passive: REA does not evaluate page JavaScript, navigate, click, close the page, or close the browser. It redacts query values and never retains headers, bodies, cookies, storage values, console argument values, or WebSocket payloads. Existing activity before attach is explicitly unavailable. See [Website observation with CDP](docs/browser-observation.md) for browser startup, schemas, limits, and the threat model.
|
|
350
|
+
|
|
330
351
|
## Roadmap
|
|
331
352
|
|
|
332
353
|
REA is growing into a toolkit for understanding software across static artifacts and observed behavior. The next capability families are:
|
|
333
354
|
|
|
334
355
|
1. **Artifact decomposition** — DMG, ASAR, ZIP, packages, universal-binary slices, application resources, embedded frameworks, mobile packages, and artifact graphs.
|
|
335
|
-
2. **Web and Electron investigation** —
|
|
356
|
+
2. **Web and Electron investigation** — extend the shipped passive CDP observation with approved interaction, screenshots, Electron IPC, route discovery, controlled replay, and visual or structural differences.
|
|
336
357
|
3. **Deterministic behavior harnesses** — stronger process-tree ownership, protocol fixtures, network policy, filesystem tracing, signals, reconnects, and cross-version comparison.
|
|
337
358
|
4. **JavaScript and source recovery** — bundle indexing, AST/module reconstruction, source-map discovery, historical-source matching, and CodeDB-backed cross-references.
|
|
338
359
|
5. **Runtime observation** — approval-gated LLDB, Frida, system logs, process and filesystem observers, and native API tracing.
|
|
@@ -377,8 +398,9 @@ flowchart LR
|
|
|
377
398
|
Router --> Hopper["Hopper provider"]
|
|
378
399
|
Router --> Native["Native macOS provider"]
|
|
379
400
|
Router --> Artifact["Artifact graph provider"]
|
|
401
|
+
Router --> Browser["Browser CDP provider"]
|
|
380
402
|
Router --> Process["Process capture provider"]
|
|
381
|
-
Router -. roadmap .-> More["
|
|
403
|
+
Router -. roadmap .-> More["Dynamic and additional<br/>static providers"]
|
|
382
404
|
Hopper --> Target["Target software"]
|
|
383
405
|
Process --> Target
|
|
384
406
|
Native --> Target
|
|
@@ -420,6 +442,27 @@ rea mcp
|
|
|
420
442
|
|
|
421
443
|
REA accepts a Mac `.app` folder directly. If an agent cannot find an app by name, tell it where the app is installed.
|
|
422
444
|
|
|
445
|
+
### CLI exit status
|
|
446
|
+
|
|
447
|
+
| Status | Meaning |
|
|
448
|
+
| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
449
|
+
| `0` | The requested operation completed. Truthful unknowns, warnings, and bounded or truncated evidence remain successful results. |
|
|
450
|
+
| `1` | Arguments, policy, permission, provider analysis, integrity checking, cancellation, timeout, setup, diagnostics, update, uninstall, output encoding, or output writing prevented completion. Structured output identifies the failure category when REA could encode it. |
|
|
451
|
+
| `128 + N` | The process ended from signal `N`, where the shell or runtime preserves the conventional signal-derived status. |
|
|
452
|
+
|
|
453
|
+
`setup` returns `1` for `planned`, `needs_confirmation`, or `needs_human`
|
|
454
|
+
because configuration is not ready; rerun it after approval or remediation.
|
|
455
|
+
`doctor` returns `1` when any check is unhealthy. Output format, full envelopes,
|
|
456
|
+
filters, and token controls never change the operation status.
|
|
457
|
+
|
|
458
|
+
When REA feeds a shell pipeline, enable `pipefail` so a downstream formatter
|
|
459
|
+
cannot hide its failure:
|
|
460
|
+
|
|
461
|
+
```bash
|
|
462
|
+
set -o pipefail
|
|
463
|
+
rea inventory-artifact ./app.asar --json | jq . > inventory.json
|
|
464
|
+
```
|
|
465
|
+
|
|
423
466
|
## Current Hopper provider
|
|
424
467
|
|
|
425
468
|
REA starts Hopper when needed; Hopper does not need to be running first. Hopper's launcher internally activates the application, so opening a target may bring Hopper to the foreground. REA asks macOS to start Hopper hidden and in the background when possible, but cannot guarantee that it will remain behind the current application.
|
package/bridge/hopper_bridge.py
CHANGED
|
@@ -628,16 +628,23 @@ def _dispatch(method, params):
|
|
|
628
628
|
global _selected_document
|
|
629
629
|
if method == "health":
|
|
630
630
|
return {"name": "REA Hopper bridge", "version": "1.0.0", "run_id": REA_RUN_ID}
|
|
631
|
-
if method
|
|
631
|
+
if method in ("shutdown", "shutdown_document"):
|
|
632
632
|
document = _session_document()
|
|
633
633
|
if document is None:
|
|
634
634
|
return {"shutdown": True, "analysis_stopped": True, "document_closed": True}
|
|
635
635
|
if document.backgroundProcessActive():
|
|
636
636
|
document.requestBackgroundProcessStop()
|
|
637
|
+
if method == "shutdown" and REA_OWNS_PROCESS_LIFETIME:
|
|
638
|
+
return {
|
|
639
|
+
"shutdown": True,
|
|
640
|
+
"analysis_stopped": not document.backgroundProcessActive(),
|
|
641
|
+
"document_closed": False,
|
|
642
|
+
"cleanup_required": True,
|
|
643
|
+
}
|
|
637
644
|
if document.backgroundProcessActive():
|
|
638
645
|
document.waitForBackgroundProcessToEnd()
|
|
639
|
-
analysis_stopped = not document.backgroundProcessActive()
|
|
640
646
|
document.closeDocument()
|
|
647
|
+
analysis_stopped = not document.backgroundProcessActive()
|
|
641
648
|
document_closed = _session_document() is None
|
|
642
649
|
return {
|
|
643
650
|
"shutdown": True,
|
|
@@ -806,7 +813,9 @@ def _serve_connection(connection):
|
|
|
806
813
|
if not isinstance(request["token"], str) or not hmac.compare_digest(request["token"], REA_TOKEN):
|
|
807
814
|
raise PermissionError("Invalid bridge capability")
|
|
808
815
|
result = _dispatch(request["method"], request["params"])
|
|
809
|
-
should_stop = request["method"] == "
|
|
816
|
+
should_stop = request["method"] == "shutdown_document" or (
|
|
817
|
+
request["method"] == "shutdown" and not result.get("cleanup_required", False)
|
|
818
|
+
)
|
|
810
819
|
response = {"id": request_id, "result": _json_safe(result)}
|
|
811
820
|
except Exception as error:
|
|
812
821
|
response = {"id": request_id if isinstance(request_id, int) else 0, "error": {"code": -32000, "message": str(error)[:512], "type": _diagnostic_type(error)}}
|
|
@@ -52,12 +52,12 @@ export class AnalysisSnapshotCache {
|
|
|
52
52
|
this.#target.format !== snapshot.target.format ||
|
|
53
53
|
this.#target.architecture !== snapshot.target.architecture))
|
|
54
54
|
this.#entries.clear();
|
|
55
|
-
this.#target = snapshot.target;
|
|
55
|
+
this.#target = structuredClone(snapshot.target);
|
|
56
56
|
let imported = 0;
|
|
57
57
|
for (const entry of snapshot.entries) {
|
|
58
58
|
if (!this.#entries.has(entry.query_id))
|
|
59
59
|
imported += 1;
|
|
60
|
-
this.#entries.set(entry.query_id, entry);
|
|
60
|
+
this.#entries.set(entry.query_id, structuredClone(entry));
|
|
61
61
|
}
|
|
62
62
|
return imported;
|
|
63
63
|
}
|
|
@@ -69,7 +69,7 @@ export class AnalysisSnapshotCache {
|
|
|
69
69
|
snapshot_version: 1,
|
|
70
70
|
target: snapshotTarget(target),
|
|
71
71
|
entries: this.entries(),
|
|
72
|
-
evidence_bundle: evidenceBundle,
|
|
72
|
+
evidence_bundle: structuredClone(evidenceBundle),
|
|
73
73
|
});
|
|
74
74
|
}
|
|
75
75
|
/** Validate target identity, merge evidence atomically, then stage entries. */
|
|
@@ -81,7 +81,9 @@ export class AnalysisSnapshotCache {
|
|
|
81
81
|
}
|
|
82
82
|
/** Return canonical entries for persistence. */
|
|
83
83
|
entries() {
|
|
84
|
-
return [...this.#entries.values()]
|
|
84
|
+
return [...this.#entries.values()]
|
|
85
|
+
.sort((left, right) => left.query_id.localeCompare(right.query_id))
|
|
86
|
+
.map((entry) => structuredClone(entry));
|
|
85
87
|
}
|
|
86
88
|
/** Replay an exact provider-specific query, marking its cached provenance. */
|
|
87
89
|
lookup(target, operation, parameters, provider) {
|
|
@@ -90,7 +92,7 @@ export class AnalysisSnapshotCache {
|
|
|
90
92
|
if (cached === undefined)
|
|
91
93
|
return undefined;
|
|
92
94
|
const subject = cached.execution.subject;
|
|
93
|
-
return {
|
|
95
|
+
return structuredClone({
|
|
94
96
|
result: cached.execution.result,
|
|
95
97
|
rawResult: cached.execution.raw_result,
|
|
96
98
|
provider: cached.execution.provider,
|
|
@@ -113,7 +115,7 @@ export class AnalysisSnapshotCache {
|
|
|
113
115
|
format: subject.format,
|
|
114
116
|
architecture: subject.architecture,
|
|
115
117
|
},
|
|
116
|
-
};
|
|
118
|
+
});
|
|
117
119
|
}
|
|
118
120
|
/** Record one successful immutable call unless the cache is full. */
|
|
119
121
|
record(target, operation, parameters, execution) {
|
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { extname } from "node:path";
|
|
3
3
|
import canonicalize from "canonicalize";
|
|
4
|
+
/** Index the first occurrence for each canonical logical path. */
|
|
5
|
+
export const indexOccurrencesByPath = (occurrences) => {
|
|
6
|
+
const byPath = new Map();
|
|
7
|
+
for (const occurrence of occurrences)
|
|
8
|
+
if (!byPath.has(occurrence.logical_path))
|
|
9
|
+
byPath.set(occurrence.logical_path, occurrence);
|
|
10
|
+
return byPath;
|
|
11
|
+
};
|
|
4
12
|
export const createOccurrence = (entry, path, parent) => ({
|
|
5
13
|
occurrence_id: `occ_${digestCanonical({ path, kind: entry.kind })}`,
|
|
6
14
|
artifact_id: null,
|
|
@@ -83,7 +91,7 @@ export const createArtifactNode = (input) => ({
|
|
|
83
91
|
architecture: null,
|
|
84
92
|
executable: input.executable,
|
|
85
93
|
content_state: input.contentState,
|
|
86
|
-
limitations: [],
|
|
94
|
+
limitations: [...(input.limitations ?? [])],
|
|
87
95
|
});
|
|
88
96
|
export const rootOccurrenceFor = (node, declaredSize) => ({
|
|
89
97
|
occurrence_id: `occ_${digestCanonical({ root: node.artifact_id })}`,
|
|
@@ -118,13 +126,14 @@ export const rekeyOccurrences = (rootArtifactId, occurrences) => {
|
|
|
118
126
|
};
|
|
119
127
|
export const createArtifactEdges = (rootArtifactId, occurrences, producer) => {
|
|
120
128
|
const byId = new Map(occurrences.map((item) => [item.occurrence_id, item]));
|
|
129
|
+
const byPath = indexOccurrencesByPath(occurrences);
|
|
121
130
|
const edges = [];
|
|
122
131
|
for (const occurrence of occurrences) {
|
|
123
132
|
if (occurrence.parent_occurrence_id === null ||
|
|
124
133
|
occurrence.artifact_id === null)
|
|
125
134
|
continue;
|
|
126
|
-
const mappedSource = occurrence.logical_path.endsWith(".map")
|
|
127
|
-
?
|
|
135
|
+
const mappedSource = occurrence.logical_path.toLowerCase().endsWith(".map")
|
|
136
|
+
? byPath.get(occurrence.logical_path.slice(0, -".map".length))
|
|
128
137
|
: undefined;
|
|
129
138
|
const parentArtifactId = mappedSource?.artifact_id ??
|
|
130
139
|
byId.get(occurrence.parent_occurrence_id)?.artifact_id ??
|
|
@@ -10,24 +10,33 @@ import { MachOSliceArtifactReader } from "../artifacts/MachOSliceArtifactReader.
|
|
|
10
10
|
import { NativeDmgArtifactReader } from "../artifacts/NativeDmgArtifactReader.js";
|
|
11
11
|
import { streamChunkToBuffer } from "../artifacts/StreamBytes.js";
|
|
12
12
|
import { artifactInventoryResultSchema, } from "../domain/artifactGraph.js";
|
|
13
|
-
import { classifyArtifactPath, classifyArtifactContent, createArtifactEdges, createArtifactNode, createOccurrence, createRootNode, digestCanonical, materializeDirectoryNodes, nearestParent, pageOf, rekeyOccurrences, rootOccurrenceFor, toOutputLimits, } from "./ArtifactGraphConstruction.js";
|
|
13
|
+
import { classifyArtifactPath, classifyArtifactContent, createArtifactEdges, createArtifactNode, createOccurrence, createRootNode, digestCanonical, indexOccurrencesByPath, materializeDirectoryNodes, nearestParent, pageOf, rekeyOccurrences, rootOccurrenceFor, toOutputLimits, } from "./ArtifactGraphConstruction.js";
|
|
14
14
|
const NATIVE_MOUNT_DISABLED = {
|
|
15
15
|
nativeMountApproved: false,
|
|
16
16
|
nativeMountEnabled: false,
|
|
17
17
|
};
|
|
18
|
+
const STRICT_INTEGRITY_POLICY = {
|
|
19
|
+
mode: "fail",
|
|
20
|
+
approved: false,
|
|
21
|
+
enabled: false,
|
|
22
|
+
maxMismatches: 1,
|
|
23
|
+
};
|
|
18
24
|
/** Inventory one local artifact without extracting or mounting it. */
|
|
19
25
|
export const inventoryArtifact = async (inputPath, limits, page, options = {}) => {
|
|
20
|
-
const snapshot = await scanArtifactInventory(inputPath, limits, options.signal, options.nativeMount ?? NATIVE_MOUNT_DISABLED);
|
|
26
|
+
const snapshot = await scanArtifactInventory(inputPath, limits, options.signal, options.nativeMount ?? NATIVE_MOUNT_DISABLED, options.integrity ?? STRICT_INTEGRITY_POLICY);
|
|
21
27
|
return paginateArtifactInventory(snapshot, page);
|
|
22
28
|
};
|
|
23
29
|
/** Scan an artifact once and retain the complete immutable graph for projection. */
|
|
24
|
-
export const scanArtifactInventory = async (inputPath, limits, signal, nativeMount = NATIVE_MOUNT_DISABLED) => {
|
|
30
|
+
export const scanArtifactInventory = async (inputPath, limits, signal, nativeMount = NATIVE_MOUNT_DISABLED, integrity = STRICT_INTEGRITY_POLICY) => {
|
|
25
31
|
abortIfNeeded(signal);
|
|
26
32
|
const path = await realpath(inputPath);
|
|
27
|
-
return scanCanonicalArtifactInventory(path, limits, signal, nativeMount);
|
|
33
|
+
return scanCanonicalArtifactInventory(path, limits, signal, nativeMount, integrity);
|
|
28
34
|
};
|
|
29
35
|
/** Scan one already-canonical artifact path without resolving it again. */
|
|
30
|
-
export const scanCanonicalArtifactInventory = async (path, limits, signal, nativeMount = NATIVE_MOUNT_DISABLED) => {
|
|
36
|
+
export const scanCanonicalArtifactInventory = async (path, limits, signal, nativeMount = NATIVE_MOUNT_DISABLED, integrity = STRICT_INTEGRITY_POLICY) => {
|
|
37
|
+
if (integrity.mode === "record-and-continue" &&
|
|
38
|
+
(!integrity.approved || !integrity.enabled))
|
|
39
|
+
throw new ArtifactReaderFailure("unavailable", "Integrity continuation requires explicit approval and operator policy");
|
|
31
40
|
const metadata = await lstat(path);
|
|
32
41
|
const rootFormat = await classifyRoot(path, metadata.isDirectory());
|
|
33
42
|
const rootDigest = metadata.isDirectory()
|
|
@@ -35,7 +44,7 @@ export const scanCanonicalArtifactInventory = async (path, limits, signal, nativ
|
|
|
35
44
|
: await hashReadable(createReadStream(path), limits.maxTotalBytes, signal);
|
|
36
45
|
const reader = await createReader(path, rootFormat, nativeMount, signal);
|
|
37
46
|
try {
|
|
38
|
-
const { nodes, occurrences } = await scanReader(reader, limits, signal);
|
|
47
|
+
const { nodes, occurrences, pendingContradictions } = await scanReader(reader, limits, signal, integrity);
|
|
39
48
|
materializeDirectoryNodes(occurrences, nodes);
|
|
40
49
|
const rootNode = createRootNode({
|
|
41
50
|
path,
|
|
@@ -51,6 +60,37 @@ export const scanCanonicalArtifactInventory = async (path, limits, signal, nativ
|
|
|
51
60
|
if (occurrence.parent_occurrence_id === null)
|
|
52
61
|
occurrence.parent_occurrence_id = rootOccurrence.occurrence_id;
|
|
53
62
|
occurrences.unshift(rootOccurrence);
|
|
63
|
+
const occurrenceById = new Map(occurrences.map((occurrence) => [occurrence.occurrence_id, occurrence]));
|
|
64
|
+
const occurrenceByPath = indexOccurrencesByPath(occurrences);
|
|
65
|
+
const integrityContradictions = pendingContradictions.map((contradiction) => {
|
|
66
|
+
const occurrence = occurrenceByPath.get(contradiction.logicalPath);
|
|
67
|
+
if (occurrence === undefined)
|
|
68
|
+
throw new ArtifactReaderFailure("integrity", "Integrity contradiction lost its graph occurrence");
|
|
69
|
+
const parent = occurrence.parent_occurrence_id === null
|
|
70
|
+
? undefined
|
|
71
|
+
: occurrenceById.get(occurrence.parent_occurrence_id);
|
|
72
|
+
const parentArtifactId = parent?.artifact_id ?? rootNode.artifact_id;
|
|
73
|
+
return {
|
|
74
|
+
contradiction_id: `ic_${digestCanonical({
|
|
75
|
+
root_artifact_id: rootNode.artifact_id,
|
|
76
|
+
logical_path: contradiction.logicalPath,
|
|
77
|
+
declared_sha256: contradiction.declaredSha256,
|
|
78
|
+
observed_sha256: contradiction.observedSha256,
|
|
79
|
+
})}`,
|
|
80
|
+
occurrence_id: occurrence.occurrence_id,
|
|
81
|
+
parent_artifact_id: parentArtifactId,
|
|
82
|
+
logical_path: contradiction.logicalPath,
|
|
83
|
+
declared_sha256: contradiction.declaredSha256,
|
|
84
|
+
observed_sha256: contradiction.observedSha256,
|
|
85
|
+
entry_kind: contradiction.entryKind,
|
|
86
|
+
unpacked: contradiction.unpacked,
|
|
87
|
+
trust: "observed-untrusted",
|
|
88
|
+
provenance: "container-integrity-metadata-versus-observed-bytes",
|
|
89
|
+
limitations: [
|
|
90
|
+
"Observed bytes contradict declared integrity metadata and cannot support equivalence.",
|
|
91
|
+
],
|
|
92
|
+
};
|
|
93
|
+
});
|
|
54
94
|
const edges = createArtifactEdges(rootNode.artifact_id, occurrences, reader?.provenance()[0]);
|
|
55
95
|
const orderedNodes = [...nodes.values()].sort((left, right) => left.artifact_id.localeCompare(right.artifact_id));
|
|
56
96
|
const orderedOccurrences = occurrences.sort((left, right) => left.logical_path.localeCompare(right.logical_path, "en"));
|
|
@@ -65,6 +105,7 @@ export const scanCanonicalArtifactInventory = async (path, limits, signal, nativ
|
|
|
65
105
|
nodes: orderedNodes,
|
|
66
106
|
occurrences: orderedOccurrences,
|
|
67
107
|
edges: orderedEdges,
|
|
108
|
+
integrity_contradictions: integrityContradictions,
|
|
68
109
|
});
|
|
69
110
|
const manifestId = `agm_${digestCanonical({
|
|
70
111
|
schema_version: 1,
|
|
@@ -88,7 +129,15 @@ export const scanCanonicalArtifactInventory = async (path, limits, signal, nativ
|
|
|
88
129
|
edges: orderedEdges,
|
|
89
130
|
limits: toOutputLimits(limits),
|
|
90
131
|
provenance: reader?.provenance() ?? [],
|
|
91
|
-
|
|
132
|
+
integrity_contradictions: integrityContradictions,
|
|
133
|
+
limitations: [
|
|
134
|
+
...inventoryLimitations(rootFormat, reader),
|
|
135
|
+
...(integrityContradictions.length === 0
|
|
136
|
+
? []
|
|
137
|
+
: [
|
|
138
|
+
`${String(integrityContradictions.length)} integrity contradiction(s) were recorded; mismatched content is observed-untrusted.`,
|
|
139
|
+
]),
|
|
140
|
+
],
|
|
92
141
|
};
|
|
93
142
|
}
|
|
94
143
|
finally {
|
|
@@ -149,11 +198,11 @@ const visitNestedAsar = async (adapterKey, logicalPath, visit) => {
|
|
|
149
198
|
const isExpandableAsar = (entry, logicalPath) => entry.kind === "file" &&
|
|
150
199
|
logicalPath.toLowerCase().endsWith(".asar") &&
|
|
151
200
|
entry.adapterKey.startsWith("/");
|
|
152
|
-
const emptyScan = () => ({ nodes: new Map(), occurrences: [] });
|
|
153
|
-
const scanReader = async (reader, limits, signal) => {
|
|
154
|
-
const { nodes, occurrences } = emptyScan();
|
|
201
|
+
const emptyScan = () => ({ nodes: new Map(), occurrences: [], pendingContradictions: [] });
|
|
202
|
+
const scanReader = async (reader, limits, signal, integrity = STRICT_INTEGRITY_POLICY) => {
|
|
203
|
+
const { nodes, occurrences, pendingContradictions } = emptyScan();
|
|
155
204
|
if (reader === undefined)
|
|
156
|
-
return { nodes, occurrences };
|
|
205
|
+
return { nodes, occurrences, pendingContradictions };
|
|
157
206
|
const occurrenceByPath = new Map();
|
|
158
207
|
const registry = new ArtifactPathRegistry();
|
|
159
208
|
let totalBytes = 0;
|
|
@@ -167,24 +216,44 @@ const scanReader = async (reader, limits, signal) => {
|
|
|
167
216
|
throw new ArtifactReaderFailure("limit", "Declared artifact bytes exceed remaining cumulative limit");
|
|
168
217
|
const digest = await hashReadable(await currentReader.open(entry, signal), Math.min(limits.maxEntryBytes, remainingBytes), signal);
|
|
169
218
|
totalBytes += digest.bytes;
|
|
170
|
-
|
|
219
|
+
const mismatched = entry.declaredSha256 !== null && entry.declaredSha256 !== digest.sha256;
|
|
220
|
+
if (mismatched && integrity.mode === "fail")
|
|
171
221
|
throw new ArtifactReaderFailure("integrity", `Artifact integrity metadata disagrees with content: ${logicalPath}`, undefined, {
|
|
172
222
|
logicalPath,
|
|
173
223
|
declaredSha256: entry.declaredSha256,
|
|
174
224
|
calculatedSha256: digest.sha256,
|
|
175
225
|
unpacked: entry.unpacked,
|
|
176
226
|
});
|
|
227
|
+
if (mismatched && entry.declaredSha256 !== null) {
|
|
228
|
+
if (pendingContradictions.length >= integrity.maxMismatches)
|
|
229
|
+
throw new ArtifactReaderFailure("limit", "Artifact integrity mismatch limit exceeded");
|
|
230
|
+
pendingContradictions.push({
|
|
231
|
+
logicalPath,
|
|
232
|
+
declaredSha256: entry.declaredSha256,
|
|
233
|
+
observedSha256: digest.sha256,
|
|
234
|
+
entryKind: entry.kind,
|
|
235
|
+
unpacked: entry.unpacked,
|
|
236
|
+
});
|
|
237
|
+
}
|
|
177
238
|
const classified = entry.kind === "slice"
|
|
178
239
|
? { kind: "universal-slice", format: "mach-o" }
|
|
179
240
|
: classifyArtifactContent(logicalPath, digest.prefix);
|
|
180
|
-
return
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
241
|
+
return {
|
|
242
|
+
node: createArtifactNode({
|
|
243
|
+
sha256: digest.sha256,
|
|
244
|
+
size: digest.bytes,
|
|
245
|
+
kind: classified.kind,
|
|
246
|
+
format: classified.format,
|
|
247
|
+
executable: entry.executable,
|
|
248
|
+
contentState: "embedded",
|
|
249
|
+
limitations: mismatched
|
|
250
|
+
? [
|
|
251
|
+
"Observed content contradicts declared integrity metadata and is untrusted.",
|
|
252
|
+
]
|
|
253
|
+
: [],
|
|
254
|
+
}),
|
|
255
|
+
mismatched,
|
|
256
|
+
};
|
|
188
257
|
};
|
|
189
258
|
const visit = async (currentReader, prefix) => {
|
|
190
259
|
for await (const entry of currentReader.entries(signal)) {
|
|
@@ -196,20 +265,24 @@ const scanReader = async (reader, limits, signal) => {
|
|
|
196
265
|
preflightEntry(entry, limits);
|
|
197
266
|
const parent = nearestParent(logicalPath, occurrenceByPath);
|
|
198
267
|
const occurrence = createOccurrence(entry, logicalPath, parent?.occurrence_id ?? null);
|
|
199
|
-
const
|
|
200
|
-
if (
|
|
201
|
-
nodes.set(node.artifact_id, node);
|
|
202
|
-
occurrence.artifact_id = node.artifact_id;
|
|
203
|
-
occurrence.hash_status =
|
|
268
|
+
const digested = await digestEntry(currentReader, entry, logicalPath);
|
|
269
|
+
if (digested !== undefined) {
|
|
270
|
+
nodes.set(digested.node.artifact_id, digested.node);
|
|
271
|
+
occurrence.artifact_id = digested.node.artifact_id;
|
|
272
|
+
occurrence.hash_status = digested.mismatched
|
|
273
|
+
? "mismatched"
|
|
274
|
+
: "verified";
|
|
275
|
+
if (digested.mismatched)
|
|
276
|
+
occurrence.limitations.push("Declared integrity metadata contradicts observed bytes.");
|
|
204
277
|
}
|
|
205
278
|
occurrences.push(occurrence);
|
|
206
279
|
occurrenceByPath.set(logicalPath, occurrence);
|
|
207
|
-
if (expandableAsar)
|
|
280
|
+
if (expandableAsar && digested?.mismatched !== true)
|
|
208
281
|
await visitNestedAsar(entry.adapterKey, logicalPath, visit);
|
|
209
282
|
}
|
|
210
283
|
};
|
|
211
284
|
await visit(reader, "");
|
|
212
|
-
return { nodes, occurrences };
|
|
285
|
+
return { nodes, occurrences, pendingContradictions };
|
|
213
286
|
};
|
|
214
287
|
const classifyRoot = async (path, directory) => {
|
|
215
288
|
if (directory)
|
|
@@ -2,15 +2,16 @@ import { realpath } from "node:fs/promises";
|
|
|
2
2
|
import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
|
|
3
3
|
import { withinRoot } from "../reference/ReferenceSourceReaderPaths.js";
|
|
4
4
|
import { scanCanonicalArtifactInventory, } from "./ArtifactInventory.js";
|
|
5
|
+
import { canonicalizeConfiguredRoots } from "./ConfiguredRoots.js";
|
|
5
6
|
/** Resolve, authorize, and scan one artifact without a second path resolution. */
|
|
6
|
-
export const scanAuthorizedArtifactInventory = async (inputPath, roots, limits, signal) => {
|
|
7
|
+
export const scanAuthorizedArtifactInventory = async (inputPath, roots, limits, signal, integrity) => {
|
|
7
8
|
if (roots.length === 0)
|
|
8
9
|
throw new ArtifactReaderFailure("path", "Artifact input roots are disabled");
|
|
9
10
|
const [path, approved] = await Promise.all([
|
|
10
11
|
realpath(inputPath),
|
|
11
|
-
|
|
12
|
+
canonicalizeConfiguredRoots(roots),
|
|
12
13
|
]);
|
|
13
14
|
if (!approved.some((root) => withinRoot(root, path)))
|
|
14
15
|
throw new ArtifactReaderFailure("path", "Artifact path is outside approved input roots");
|
|
15
|
-
return scanCanonicalArtifactInventory(path, limits, signal);
|
|
16
|
+
return scanCanonicalArtifactInventory(path, limits, signal, undefined, integrity);
|
|
16
17
|
};
|