rea-agents 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +124 -20
- package/bridge/hopper_bridge.py +141 -13
- package/dist/application/AnalysisProvider.js +12 -1
- package/dist/application/ArtifactExtraction.js +166 -0
- package/dist/application/ArtifactGraphConstruction.js +257 -0
- package/dist/application/ArtifactInventory.js +253 -0
- package/dist/application/BinarySession.js +164 -14
- package/dist/application/CompositeProvider.js +73 -0
- package/dist/application/DirectAnalysis.js +27 -3
- package/dist/application/Doctor.js +35 -6
- package/dist/application/EnhancedTools.js +8 -7
- package/dist/application/EvidenceBundleCommands.js +29 -0
- package/dist/application/EvidenceBundleFiles.js +127 -0
- package/dist/application/EvidenceLedger.js +225 -18
- package/dist/application/FilesystemSnapshot.js +124 -0
- package/dist/application/LinuxHopper.js +186 -0
- package/dist/application/LoopbackReplay.js +195 -37
- package/dist/application/ProcessHarness.js +200 -191
- package/dist/application/ProcessNormalization.js +44 -0
- package/dist/application/ProcessOwnership.js +84 -0
- package/dist/application/ProcessSampling.js +284 -0
- package/dist/application/RealHopperAssertions.js +105 -0
- package/dist/application/ReferenceSourceImport.js +182 -0
- package/dist/application/ReferenceSourceImportEntries.js +122 -0
- package/dist/application/ReferenceSourceImportPolicy.js +73 -0
- package/dist/application/ReferenceSourceImportTypes.js +18 -0
- package/dist/application/ReferenceSourceVcsAdapter.js +34 -0
- package/dist/application/Setup.js +192 -41
- package/dist/application/Uninstall.js +130 -0
- package/dist/application/runtime.js +8 -1
- package/dist/artifacts/ArtifactPaths.js +51 -0
- package/dist/artifacts/ArtifactProvider.js +130 -0
- package/dist/artifacts/ArtifactReader.js +9 -0
- package/dist/artifacts/AsarArtifactReader.js +62 -0
- package/dist/artifacts/DirectoryArtifactReader.js +107 -0
- package/dist/artifacts/MachOSliceArtifactReader.js +66 -0
- package/dist/artifacts/SafeOutputTree.js +199 -0
- package/dist/artifacts/StreamBytes.js +10 -0
- package/dist/artifacts/ZipArtifactReader.js +109 -0
- package/dist/cli.js +153 -6
- package/dist/config.js +35 -1
- package/dist/contracts/artifactComparisonExample.js +95 -0
- package/dist/contracts/artifactToolContracts.js +84 -0
- package/dist/contracts/enhancedInputs.js +4 -0
- package/dist/contracts/functionComparisonExample.js +57 -0
- package/dist/contracts/investigationExamples.js +118 -0
- package/dist/contracts/nativeToolContracts.js +52 -0
- package/dist/contracts/processCaptureExample.js +25 -0
- package/dist/contracts/toolContractExamples.js +69 -0
- package/dist/contracts/toolContracts.js +99 -36
- package/dist/contracts/toolOutputSchemas.js +154 -82
- package/dist/contracts/unknownContractExamples.js +33 -0
- package/dist/domain/artifactComparison.js +273 -0
- package/dist/domain/artifactGraph.js +194 -0
- package/dist/domain/artifactInventoryEvidence.js +150 -0
- package/dist/domain/binaryTarget.js +55 -0
- package/dist/domain/bundleComparison.js +266 -0
- package/dist/domain/callPath.js +346 -0
- package/dist/domain/changedBehavior.js +294 -0
- package/dist/domain/errors.js +192 -4
- package/dist/domain/evidence.js +41 -10
- package/dist/domain/evidenceBundle.js +187 -6
- package/dist/domain/functionComparison.js +201 -0
- package/dist/domain/functionComparisonNormalization.js +112 -0
- package/dist/domain/functionComparisonResults.js +54 -0
- package/dist/domain/functionComparisonSchemas.js +82 -0
- package/dist/domain/functionDossierEvidence.js +171 -0
- package/dist/domain/hopperValues.js +14 -7
- package/dist/domain/nativeInspection.js +142 -0
- package/dist/domain/processCapture.js +152 -54
- package/dist/domain/processComparison.js +106 -0
- package/dist/domain/reconstructionUnknowns.js +90 -0
- package/dist/domain/reconstructionVerification.js +285 -0
- package/dist/domain/reconstructionVerificationSchemas.js +126 -0
- package/dist/domain/referenceSourceClassification.js +496 -0
- package/dist/domain/referenceSourceGraph.js +376 -0
- package/dist/domain/referenceSourceImportParsing.js +235 -0
- package/dist/domain/residualUnknown.js +239 -0
- package/dist/domain/staticRuntimeCorrelation.js +375 -0
- package/dist/hopper/BridgeLauncher.js +40 -3
- package/dist/hopper/HopperClient.js +14 -5
- package/dist/hopper/HopperProvider.js +57 -22
- package/dist/identity.js +1 -0
- package/dist/main.js +5 -1
- package/dist/native/CommandRunner.js +156 -0
- package/dist/native/NativeMacOSProvider.js +306 -0
- package/dist/native/NativeMachoInspection.js +135 -0
- package/dist/native/parsers/codesign.js +55 -0
- package/dist/native/parsers/demangle.js +26 -0
- package/dist/native/parsers/dyldInfo.js +25 -0
- package/dist/native/parsers/lipo.js +67 -0
- package/dist/native/parsers/otool.js +193 -0
- package/dist/native/parsers/plist.js +23 -0
- package/dist/reference/ReferenceSourceReader.js +73 -0
- package/dist/reference/ReferenceSourceReaderEntries.js +206 -0
- package/dist/reference/ReferenceSourceReaderErrors.js +19 -0
- package/dist/reference/ReferenceSourceReaderFile.js +119 -0
- package/dist/reference/ReferenceSourceReaderPaths.js +23 -0
- package/dist/reference/ReferenceSourceReaderTypes.js +2 -0
- package/dist/reference/ReferenceSourceReaderValidate.js +71 -0
- package/dist/server/createServer.js +31 -5
- package/dist/server/recordDerivedEvidence.js +10 -0
- package/dist/server/registerArtifactComparisonTool.js +62 -0
- package/dist/server/registerArtifactTools.js +6 -0
- package/dist/server/registerBundleComparisonTool.js +47 -0
- package/dist/server/registerEnhancedTools.js +64 -12
- package/dist/server/registerEvidenceTools.js +36 -0
- package/dist/server/registerFunctionComparisonTool.js +68 -0
- package/dist/server/registerInvestigationTools.js +224 -0
- package/dist/server/registerNativeTools.js +6 -0
- package/dist/server/registerOfficialTools.js +47 -14
- package/dist/server/registerProcessComparisonTool.js +106 -0
- package/dist/server/registerSessionTools.js +179 -70
- package/dist/server/sessionEvidence.js +28 -0
- package/dist/server/sessionToolPolicies.js +64 -0
- package/dist/server/toolRegistrationOptions.js +7 -0
- package/dist/server/toolResult.js +8 -5
- package/install.sh +198 -0
- package/package.json +19 -2
- package/scripts/rea.mjs +5 -1
- package/skills/rea-analysis/SKILL.md +77 -2
- /package/dist/{application/HopperToolPort.js → domain/referenceSourcePolicy.js} +0 -0
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { readFile } from "node:fs/promises";
|
|
3
|
+
import { promisify } from "node:util";
|
|
4
|
+
const execFileAsync = promisify(execFile);
|
|
5
|
+
const PROC_READ_CONCURRENCY = 64;
|
|
6
|
+
const isExpectedProcReadFailure = (cause) => cause instanceof Error &&
|
|
7
|
+
"code" in cause &&
|
|
8
|
+
(cause.code === "ENOENT" ||
|
|
9
|
+
cause.code === "ESRCH" ||
|
|
10
|
+
cause.code === "EACCES" ||
|
|
11
|
+
cause.code === "EPERM");
|
|
12
|
+
const handleProcReadFailure = (cause) => {
|
|
13
|
+
if (isExpectedProcReadFailure(cause))
|
|
14
|
+
return undefined;
|
|
15
|
+
throw cause;
|
|
16
|
+
};
|
|
17
|
+
const parseProcStat = (identifier, stat) => {
|
|
18
|
+
const commandEnd = stat.lastIndexOf(")");
|
|
19
|
+
if (commandEnd < 0)
|
|
20
|
+
return undefined;
|
|
21
|
+
const fields = stat
|
|
22
|
+
.slice(commandEnd + 2)
|
|
23
|
+
.trim()
|
|
24
|
+
.split(/\s+/);
|
|
25
|
+
const pid = Number(identifier);
|
|
26
|
+
const parentPid = Number(fields[1]);
|
|
27
|
+
const startTime = fields[19];
|
|
28
|
+
if (!Number.isSafeInteger(pid) ||
|
|
29
|
+
pid <= 0 ||
|
|
30
|
+
!Number.isSafeInteger(parentPid) ||
|
|
31
|
+
parentPid < 0 ||
|
|
32
|
+
startTime === undefined ||
|
|
33
|
+
!/^\d+$/.test(startTime))
|
|
34
|
+
return undefined;
|
|
35
|
+
return {
|
|
36
|
+
pid,
|
|
37
|
+
parent_pid: parentPid,
|
|
38
|
+
command: "",
|
|
39
|
+
startTime,
|
|
40
|
+
};
|
|
41
|
+
};
|
|
42
|
+
const readProcessStat = async (pid, signal) => {
|
|
43
|
+
try {
|
|
44
|
+
const stat = await readFile(`/proc/${String(pid)}/stat`, {
|
|
45
|
+
encoding: "utf8",
|
|
46
|
+
signal,
|
|
47
|
+
});
|
|
48
|
+
return parseProcStat(String(pid), stat);
|
|
49
|
+
}
|
|
50
|
+
catch (cause) {
|
|
51
|
+
return handleProcReadFailure(cause);
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
const readProcessCommand = async (pid, signal) => {
|
|
55
|
+
try {
|
|
56
|
+
return await readFile(`/proc/${String(pid)}/cmdline`, { signal });
|
|
57
|
+
}
|
|
58
|
+
catch (cause) {
|
|
59
|
+
return handleProcReadFailure(cause);
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
const readLinuxChildren = async (pid, signal) => {
|
|
63
|
+
let text;
|
|
64
|
+
try {
|
|
65
|
+
text = await readFile(`/proc/${String(pid)}/task/${String(pid)}/children`, {
|
|
66
|
+
encoding: "utf8",
|
|
67
|
+
signal,
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
catch (cause) {
|
|
71
|
+
return handleProcReadFailure(cause);
|
|
72
|
+
}
|
|
73
|
+
return text
|
|
74
|
+
.trim()
|
|
75
|
+
.split(/\s+/)
|
|
76
|
+
.filter((value) => /^\d+$/.test(value))
|
|
77
|
+
.map(Number)
|
|
78
|
+
.filter((child) => Number.isSafeInteger(child) && child > 0);
|
|
79
|
+
};
|
|
80
|
+
const inspectProcess = async (pid, expectedParent, signal, identities) => {
|
|
81
|
+
const before = await readProcessStat(pid, signal);
|
|
82
|
+
if (before === undefined || before.startTime === undefined)
|
|
83
|
+
return undefined;
|
|
84
|
+
const existing = identities.get(pid);
|
|
85
|
+
if (existing !== undefined && existing !== before.startTime)
|
|
86
|
+
return undefined;
|
|
87
|
+
if (expectedParent !== undefined && before.parent_pid !== expectedParent)
|
|
88
|
+
return undefined;
|
|
89
|
+
const [commandBuffer, children] = await Promise.all([
|
|
90
|
+
readProcessCommand(pid, signal),
|
|
91
|
+
readLinuxChildren(pid, signal),
|
|
92
|
+
]);
|
|
93
|
+
const after = await readProcessStat(pid, signal);
|
|
94
|
+
if (after === undefined || after.startTime === undefined)
|
|
95
|
+
return undefined;
|
|
96
|
+
if (before.startTime !== after.startTime ||
|
|
97
|
+
before.parent_pid !== after.parent_pid)
|
|
98
|
+
return undefined;
|
|
99
|
+
if (expectedParent !== undefined && after.parent_pid !== expectedParent)
|
|
100
|
+
return undefined;
|
|
101
|
+
const command = commandBuffer === undefined
|
|
102
|
+
? ""
|
|
103
|
+
: commandBuffer.toString().replaceAll("\0", " ").trim();
|
|
104
|
+
return {
|
|
105
|
+
pid: before.pid,
|
|
106
|
+
parent_pid: before.parent_pid,
|
|
107
|
+
startTime: before.startTime,
|
|
108
|
+
command,
|
|
109
|
+
children: children ?? [],
|
|
110
|
+
};
|
|
111
|
+
};
|
|
112
|
+
const sampleLinux = async (context) => {
|
|
113
|
+
const { rootPid, limit, signal, sampledPids, identities } = context;
|
|
114
|
+
if (limit <= 0)
|
|
115
|
+
return [];
|
|
116
|
+
const rootStat = await readProcessStat(rootPid, signal);
|
|
117
|
+
if (rootStat === undefined)
|
|
118
|
+
return [];
|
|
119
|
+
const rootChildren = await readLinuxChildren(rootPid, signal);
|
|
120
|
+
if (rootChildren === undefined)
|
|
121
|
+
return samplePs(context);
|
|
122
|
+
const rows = [];
|
|
123
|
+
const visited = new Set([rootPid]);
|
|
124
|
+
let level = [{ pid: rootPid, expectedParent: undefined }];
|
|
125
|
+
while (level.length > 0 && rows.length < limit) {
|
|
126
|
+
const batchSize = Math.min(level.length, PROC_READ_CONCURRENCY);
|
|
127
|
+
const batch = level.slice(0, batchSize);
|
|
128
|
+
level = level.slice(batchSize);
|
|
129
|
+
const batchResults = await Promise.all(batch.map((item) => inspectProcess(item.pid, item.expectedParent, signal, identities)));
|
|
130
|
+
const nextLevel = [];
|
|
131
|
+
for (const node of batchResults) {
|
|
132
|
+
if (node === undefined)
|
|
133
|
+
continue;
|
|
134
|
+
if (!sampledPids.has(node.pid)) {
|
|
135
|
+
if (rows.length >= limit)
|
|
136
|
+
break;
|
|
137
|
+
rows.push(node);
|
|
138
|
+
}
|
|
139
|
+
if (rows.length >= limit)
|
|
140
|
+
break;
|
|
141
|
+
for (const child of node.children) {
|
|
142
|
+
if (visited.has(child))
|
|
143
|
+
continue;
|
|
144
|
+
visited.add(child);
|
|
145
|
+
nextLevel.push({ pid: child, expectedParent: node.pid });
|
|
146
|
+
}
|
|
147
|
+
if (rows.length >= limit)
|
|
148
|
+
break;
|
|
149
|
+
}
|
|
150
|
+
level.push(...nextLevel);
|
|
151
|
+
}
|
|
152
|
+
return rows;
|
|
153
|
+
};
|
|
154
|
+
const readPsRows = async (signal) => {
|
|
155
|
+
const { stdout } = await execFileAsync("ps", ["-axo", "pid=,ppid=,command="], { signal });
|
|
156
|
+
return stdout
|
|
157
|
+
.split("\n")
|
|
158
|
+
.map((line) => /\s*(\d+)\s+(\d+)\s+(.*)/u.exec(line))
|
|
159
|
+
.filter((match) => match !== null)
|
|
160
|
+
.map((match) => ({
|
|
161
|
+
pid: Number(match[1]),
|
|
162
|
+
parent_pid: Number(match[2]),
|
|
163
|
+
command: match[3] ?? "",
|
|
164
|
+
startTime: undefined,
|
|
165
|
+
}))
|
|
166
|
+
.filter((row) => Number.isSafeInteger(row.pid) &&
|
|
167
|
+
row.pid > 0 &&
|
|
168
|
+
Number.isSafeInteger(row.parent_pid) &&
|
|
169
|
+
row.parent_pid >= 0)
|
|
170
|
+
.sort((left, right) => left.pid - right.pid);
|
|
171
|
+
};
|
|
172
|
+
const samplePs = async (context) => {
|
|
173
|
+
const { rootPid, limit, signal, sampledPids } = context;
|
|
174
|
+
if (limit <= 0)
|
|
175
|
+
return [];
|
|
176
|
+
const rows = await readPsRows(signal);
|
|
177
|
+
const rowByPid = new Map(rows.map((row) => [row.pid, row]));
|
|
178
|
+
const childrenByParent = new Map();
|
|
179
|
+
for (const row of rows) {
|
|
180
|
+
const siblings = childrenByParent.get(row.parent_pid) ?? [];
|
|
181
|
+
siblings.push(row);
|
|
182
|
+
childrenByParent.set(row.parent_pid, siblings);
|
|
183
|
+
}
|
|
184
|
+
const result = [];
|
|
185
|
+
const visited = new Set();
|
|
186
|
+
const queue = [rootPid];
|
|
187
|
+
let index = 0;
|
|
188
|
+
while (index < queue.length && result.length < limit) {
|
|
189
|
+
const pid = queue[index];
|
|
190
|
+
if (pid === undefined)
|
|
191
|
+
break;
|
|
192
|
+
index += 1;
|
|
193
|
+
if (visited.has(pid))
|
|
194
|
+
continue;
|
|
195
|
+
visited.add(pid);
|
|
196
|
+
const row = rowByPid.get(pid);
|
|
197
|
+
if (row !== undefined && !sampledPids.has(row.pid)) {
|
|
198
|
+
result.push(row);
|
|
199
|
+
}
|
|
200
|
+
if (result.length >= limit)
|
|
201
|
+
break;
|
|
202
|
+
for (const child of childrenByParent.get(pid) ?? []) {
|
|
203
|
+
if (!visited.has(child.pid))
|
|
204
|
+
queue.push(child.pid);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
return result;
|
|
208
|
+
};
|
|
209
|
+
const sampleProcesses = async (context) => {
|
|
210
|
+
if (process.platform === "win32" || context.limit <= 0)
|
|
211
|
+
return [];
|
|
212
|
+
const rows = process.platform === "linux"
|
|
213
|
+
? await sampleLinux(context)
|
|
214
|
+
: await samplePs(context);
|
|
215
|
+
const { elapsedMs, sampledPids, identities } = context;
|
|
216
|
+
const samples = [];
|
|
217
|
+
for (const row of rows) {
|
|
218
|
+
if (sampledPids.has(row.pid))
|
|
219
|
+
continue;
|
|
220
|
+
if (row.startTime !== undefined) {
|
|
221
|
+
const existing = identities.get(row.pid);
|
|
222
|
+
if (existing !== undefined && existing !== row.startTime)
|
|
223
|
+
continue;
|
|
224
|
+
identities.set(row.pid, row.startTime);
|
|
225
|
+
}
|
|
226
|
+
samples.push({
|
|
227
|
+
at_ms: elapsedMs,
|
|
228
|
+
pid: row.pid,
|
|
229
|
+
parent_pid: row.parent_pid,
|
|
230
|
+
command: row.command,
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
return samples;
|
|
234
|
+
};
|
|
235
|
+
/** Start bounded process-tree sampling and expose an awaited stop. */
|
|
236
|
+
export const startProcessSampler = (rootPid, started, limit, samples) => {
|
|
237
|
+
const identities = new Map();
|
|
238
|
+
const sampledPids = new Set(samples.map(({ pid }) => pid));
|
|
239
|
+
let pending;
|
|
240
|
+
let stopped = false;
|
|
241
|
+
let abortCurrent;
|
|
242
|
+
let partial = false;
|
|
243
|
+
const sample = () => {
|
|
244
|
+
if (stopped || pending !== undefined || sampledPids.size >= limit)
|
|
245
|
+
return;
|
|
246
|
+
const controller = new AbortController();
|
|
247
|
+
abortCurrent = () => controller.abort();
|
|
248
|
+
pending = sampleProcesses({
|
|
249
|
+
rootPid,
|
|
250
|
+
elapsedMs: Date.now() - started,
|
|
251
|
+
limit: limit - sampledPids.size,
|
|
252
|
+
sampledPids,
|
|
253
|
+
identities,
|
|
254
|
+
signal: controller.signal,
|
|
255
|
+
})
|
|
256
|
+
.then((values) => {
|
|
257
|
+
for (const value of values) {
|
|
258
|
+
if (sampledPids.size >= limit)
|
|
259
|
+
break;
|
|
260
|
+
if (sampledPids.has(value.pid))
|
|
261
|
+
continue;
|
|
262
|
+
sampledPids.add(value.pid);
|
|
263
|
+
samples.push(value);
|
|
264
|
+
}
|
|
265
|
+
})
|
|
266
|
+
.catch((cause) => {
|
|
267
|
+
if (!(cause instanceof Error && cause.name === "AbortError"))
|
|
268
|
+
partial = true;
|
|
269
|
+
})
|
|
270
|
+
.then(() => {
|
|
271
|
+
abortCurrent = undefined;
|
|
272
|
+
pending = undefined;
|
|
273
|
+
});
|
|
274
|
+
};
|
|
275
|
+
const timer = setInterval(sample, 50);
|
|
276
|
+
sample();
|
|
277
|
+
return async () => {
|
|
278
|
+
stopped = true;
|
|
279
|
+
clearInterval(timer);
|
|
280
|
+
abortCurrent?.();
|
|
281
|
+
await pending;
|
|
282
|
+
return { partial };
|
|
283
|
+
};
|
|
284
|
+
};
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
const addressSchema = z.string().regex(/^0x[0-9a-f]+$/iu);
|
|
3
|
+
const identitySchema = z.object({ address: addressSchema, name: z.string() });
|
|
4
|
+
const collectionSchema = z.object({
|
|
5
|
+
items: z.array(z.unknown()),
|
|
6
|
+
total: z.number().int().min(0).nullable(),
|
|
7
|
+
returned: z.number().int().min(0),
|
|
8
|
+
truncated: z.boolean(),
|
|
9
|
+
next_offset: z.number().int().min(0).nullable(),
|
|
10
|
+
});
|
|
11
|
+
/** Extract a real address from the provider's exact procedure page. */
|
|
12
|
+
export const firstProcedureAddress = (input) => {
|
|
13
|
+
const page = z
|
|
14
|
+
.object({
|
|
15
|
+
items: z.array(z.object({ address: addressSchema })).min(1),
|
|
16
|
+
offset: z.number().int().min(0),
|
|
17
|
+
limit: z.number().int().min(1),
|
|
18
|
+
total: z.number().int().min(1),
|
|
19
|
+
next_offset: z.number().int().min(0).nullable(),
|
|
20
|
+
has_more: z.boolean(),
|
|
21
|
+
})
|
|
22
|
+
.parse(input);
|
|
23
|
+
const first = page.items[0];
|
|
24
|
+
if (first === undefined)
|
|
25
|
+
throw new TypeError("Procedure page was empty");
|
|
26
|
+
return first.address;
|
|
27
|
+
};
|
|
28
|
+
/** Reject empty and success-shaped embedded decompilation failures. */
|
|
29
|
+
export const requirePseudocode = (input, operation) => {
|
|
30
|
+
if (typeof input !== "string" || input.trim().length === 0)
|
|
31
|
+
throw new TypeError(`${operation} returned empty pseudocode`);
|
|
32
|
+
if (input === "No output" || input.startsWith("Error:"))
|
|
33
|
+
throw new TypeError(`${operation} returned an embedded failure: ${input}`);
|
|
34
|
+
return input;
|
|
35
|
+
};
|
|
36
|
+
/** Validate an exact bounded Hopper function dossier semantically. */
|
|
37
|
+
export const requireFunctionDossier = (input, expectedAddress) => {
|
|
38
|
+
const value = objectValue(input, "analyze_function returned no function dossier");
|
|
39
|
+
const procedure = identitySchema.parse(value.procedure);
|
|
40
|
+
if (procedure.address !== expectedAddress)
|
|
41
|
+
throw new TypeError("analyze_function returned the wrong procedure");
|
|
42
|
+
if (procedure.name.trim().length === 0)
|
|
43
|
+
throw new TypeError("analyze_function omitted the procedure name");
|
|
44
|
+
validatePseudocodeBounds(value.pseudocode);
|
|
45
|
+
for (const field of COLLECTION_FIELDS)
|
|
46
|
+
validateCollection(value[field], field);
|
|
47
|
+
for (const field of ["callers", "callees"])
|
|
48
|
+
z.array(identitySchema).parse(collectionSchema.parse(value[field]).items);
|
|
49
|
+
validateBlocks(collectionSchema.parse(value.basic_blocks).items);
|
|
50
|
+
validateInstructionScan(value.instruction_scan);
|
|
51
|
+
return value;
|
|
52
|
+
};
|
|
53
|
+
/** Require every relationship to be an actual hexadecimal address. */
|
|
54
|
+
export const requireAddressArray = (input, operation) => {
|
|
55
|
+
const parsed = z.array(addressSchema).parse(input);
|
|
56
|
+
if (parsed.some((address) => address.length === 0))
|
|
57
|
+
throw new TypeError(`${operation} returned an invalid address list`);
|
|
58
|
+
return parsed;
|
|
59
|
+
};
|
|
60
|
+
const COLLECTION_FIELDS = [
|
|
61
|
+
"comments",
|
|
62
|
+
"callers",
|
|
63
|
+
"callees",
|
|
64
|
+
"incoming_references",
|
|
65
|
+
"outgoing_references",
|
|
66
|
+
"referenced_strings",
|
|
67
|
+
"referenced_names",
|
|
68
|
+
"basic_blocks",
|
|
69
|
+
];
|
|
70
|
+
const validatePseudocodeBounds = (input) => {
|
|
71
|
+
const pseudocode = z
|
|
72
|
+
.object({
|
|
73
|
+
text: z.string(),
|
|
74
|
+
returned_chars: z.number().int().min(0),
|
|
75
|
+
total_chars: z.number().int().min(0),
|
|
76
|
+
})
|
|
77
|
+
.parse(input);
|
|
78
|
+
const text = requirePseudocode(pseudocode.text, "analyze_function");
|
|
79
|
+
if (pseudocode.returned_chars !== [...text].length ||
|
|
80
|
+
pseudocode.total_chars < pseudocode.returned_chars)
|
|
81
|
+
throw new TypeError("analyze_function returned inconsistent pseudocode bounds");
|
|
82
|
+
};
|
|
83
|
+
const validateCollection = (input, field) => {
|
|
84
|
+
const collection = collectionSchema.parse(input);
|
|
85
|
+
if (collection.returned !== collection.items.length)
|
|
86
|
+
throw new TypeError(`analyze_function returned an invalid ${field} result`);
|
|
87
|
+
};
|
|
88
|
+
const validateBlocks = (items) => {
|
|
89
|
+
for (const block of items)
|
|
90
|
+
if (!Array.isArray(objectValue(block, "Invalid basic block").successors))
|
|
91
|
+
throw new TypeError("analyze_function omitted CFG successor evidence");
|
|
92
|
+
};
|
|
93
|
+
const validateInstructionScan = (input) => {
|
|
94
|
+
const scan = z
|
|
95
|
+
.object({ scanned: z.number().int().min(0), truncated: z.boolean() })
|
|
96
|
+
.parse(input);
|
|
97
|
+
if (scan.scanned === 0 && !scan.truncated)
|
|
98
|
+
throw new TypeError("analyze_function omitted instruction scan limitations");
|
|
99
|
+
};
|
|
100
|
+
const objectValue = (input, message) => {
|
|
101
|
+
const parsed = z.record(z.string(), z.unknown()).safeParse(input);
|
|
102
|
+
if (!parsed.success)
|
|
103
|
+
throw new TypeError(message);
|
|
104
|
+
return parsed.data;
|
|
105
|
+
};
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import { isSecretLikePath } from "../domain/referenceSourceClassification.js";
|
|
2
|
+
import { createHistoricalSourceGraph, } from "../domain/referenceSourceGraph.js";
|
|
3
|
+
import { err, ok } from "../domain/result.js";
|
|
4
|
+
import { readReferenceSource } from "../reference/ReferenceSourceReader.js";
|
|
5
|
+
import { parseReferenceSourceEntries } from "./ReferenceSourceImportEntries.js";
|
|
6
|
+
import { readReferenceSourceVcs } from "./ReferenceSourceVcsAdapter.js";
|
|
7
|
+
import {} from "./ReferenceSourceImportTypes.js";
|
|
8
|
+
import { prepareReferenceSourceImport } from "./ReferenceSourceImportPolicy.js";
|
|
9
|
+
const compareCodePoints = (left, right) => {
|
|
10
|
+
const leftPoints = [...left].map((value) => value.codePointAt(0) ?? 0);
|
|
11
|
+
const rightPoints = [...right].map((value) => value.codePointAt(0) ?? 0);
|
|
12
|
+
const length = Math.min(leftPoints.length, rightPoints.length);
|
|
13
|
+
for (let index = 0; index < length; index += 1) {
|
|
14
|
+
const difference = (leftPoints[index] ?? 0) - (rightPoints[index] ?? 0);
|
|
15
|
+
if (difference !== 0)
|
|
16
|
+
return difference;
|
|
17
|
+
}
|
|
18
|
+
return leftPoints.length - rightPoints.length;
|
|
19
|
+
};
|
|
20
|
+
const byCodePoint = (left, right) => compareCodePoints(left, right);
|
|
21
|
+
const failure = (code, message) => ({
|
|
22
|
+
tag: "reference-source-import",
|
|
23
|
+
code,
|
|
24
|
+
message,
|
|
25
|
+
});
|
|
26
|
+
const cancelled = () => failure("cancelled", "Reference source import cancelled");
|
|
27
|
+
const isAborted = (signal) => signal?.aborted === true;
|
|
28
|
+
const relationshipKey = (relationship) => `${relationship.from_path}\u0000${relationship.to}\u0000${relationship.kind}\u0000${relationship.resolution}\u0000${relationship.parse_state}`;
|
|
29
|
+
const deduplicateRelationships = (relationships) => {
|
|
30
|
+
const seen = new Set();
|
|
31
|
+
const result = [];
|
|
32
|
+
for (const relationship of relationships) {
|
|
33
|
+
const key = relationshipKey(relationship);
|
|
34
|
+
if (seen.has(key))
|
|
35
|
+
continue;
|
|
36
|
+
seen.add(key);
|
|
37
|
+
result.push(relationship);
|
|
38
|
+
}
|
|
39
|
+
return result;
|
|
40
|
+
};
|
|
41
|
+
const buildProvenance = (options) => ({
|
|
42
|
+
importer: options.importer ?? "rea-reference-source-import",
|
|
43
|
+
importer_version: options.importerVersion ?? null,
|
|
44
|
+
caller: options.caller,
|
|
45
|
+
});
|
|
46
|
+
const deriveLanguages = (entries) => [
|
|
47
|
+
...new Set(entries.flatMap((entry) => entry.kind === "file" && entry.language !== null
|
|
48
|
+
? [entry.language]
|
|
49
|
+
: [])),
|
|
50
|
+
].sort(byCodePoint);
|
|
51
|
+
const deriveManifests = (entries) => entries
|
|
52
|
+
.flatMap((entry) => entry.kind === "file" && entry.classifications.includes("manifest")
|
|
53
|
+
? [entry.path]
|
|
54
|
+
: [])
|
|
55
|
+
.sort(byCodePoint);
|
|
56
|
+
const deriveInventoryState = (input) => {
|
|
57
|
+
if (input.limitations.length > 0)
|
|
58
|
+
return "partial";
|
|
59
|
+
if (input.exclusions.length > 0)
|
|
60
|
+
return "partial";
|
|
61
|
+
if (input.parse_failures.length > 0)
|
|
62
|
+
return "partial";
|
|
63
|
+
const partialEntry = input.entries.some((entry) => entry.limitations.length > 0 ||
|
|
64
|
+
(entry.kind === "file" && entry.content_state !== "hashed") ||
|
|
65
|
+
(entry.kind === "directory" && entry.tree_state !== "enumerated") ||
|
|
66
|
+
(entry.kind === "symlink" && entry.target_state !== "internal"));
|
|
67
|
+
if (partialEntry)
|
|
68
|
+
return "partial";
|
|
69
|
+
const partialRelationship = input.relationships.some(({ parse_state, resolution }) => parse_state !== "parsed" ||
|
|
70
|
+
["unresolved", "unknown"].includes(resolution));
|
|
71
|
+
if (partialRelationship)
|
|
72
|
+
return "partial";
|
|
73
|
+
return "complete";
|
|
74
|
+
};
|
|
75
|
+
const sortAndLimit = (exclusions, maxExclusions) => {
|
|
76
|
+
const sorted = [...exclusions].sort((left, right) => {
|
|
77
|
+
const byPath = byCodePoint(left.path, right.path);
|
|
78
|
+
if (byPath !== 0)
|
|
79
|
+
return byPath;
|
|
80
|
+
return byCodePoint(left.reason, right.reason);
|
|
81
|
+
});
|
|
82
|
+
if (sorted.length <= maxExclusions)
|
|
83
|
+
return sorted;
|
|
84
|
+
return sorted.slice(0, maxExclusions);
|
|
85
|
+
};
|
|
86
|
+
/**
|
|
87
|
+
* Import a reference source directory into a committed historical source graph.
|
|
88
|
+
*
|
|
89
|
+
* The import is deterministic, parallel-safe, and never executes source, hooks,
|
|
90
|
+
* git subprocesses, or network requests. Secret-like paths are redacted before
|
|
91
|
+
* they are committed to the graph.
|
|
92
|
+
*/
|
|
93
|
+
export const importReferenceSource = async (options) => {
|
|
94
|
+
if (isAborted(options.signal))
|
|
95
|
+
return err(cancelled());
|
|
96
|
+
const prepared = await prepareReferenceSourceImport(options);
|
|
97
|
+
if (!prepared.ok)
|
|
98
|
+
return prepared;
|
|
99
|
+
const { ignored, limits, root, secrets } = prepared.value;
|
|
100
|
+
if (isAborted(options.signal))
|
|
101
|
+
return err(cancelled());
|
|
102
|
+
const exclusions = [];
|
|
103
|
+
const shouldExclude = (path) => {
|
|
104
|
+
if (isSecretLikePath(path) || secrets.ignores(path)) {
|
|
105
|
+
exclusions.push({ path, reason: "configured-secret" });
|
|
106
|
+
return true;
|
|
107
|
+
}
|
|
108
|
+
if (ignored.ignores(path)) {
|
|
109
|
+
exclusions.push({ path, reason: "caller-excluded" });
|
|
110
|
+
return true;
|
|
111
|
+
}
|
|
112
|
+
return false;
|
|
113
|
+
};
|
|
114
|
+
const [readResult, vcs] = await Promise.all([
|
|
115
|
+
readReferenceSource(root, limits, {
|
|
116
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
117
|
+
shouldExclude,
|
|
118
|
+
}),
|
|
119
|
+
readReferenceSourceVcs(root, options.signal),
|
|
120
|
+
]);
|
|
121
|
+
if (!readResult.ok) {
|
|
122
|
+
const error = readResult.error;
|
|
123
|
+
if (error.code === "cancelled")
|
|
124
|
+
return err(cancelled());
|
|
125
|
+
return err(failure(error.code === "invalid-limits" ? "invalid-limits" : "io", error.message));
|
|
126
|
+
}
|
|
127
|
+
if (isAborted(options.signal))
|
|
128
|
+
return err(cancelled());
|
|
129
|
+
const read = readResult.value;
|
|
130
|
+
const filePaths = new Set(read.entries
|
|
131
|
+
.filter((entry) => entry.status === "read" && entry.kind === "file")
|
|
132
|
+
.map((entry) => entry.path));
|
|
133
|
+
const { entries, relationships, parseFailures, limitations } = parseReferenceSourceEntries(read, filePaths, options.signal);
|
|
134
|
+
if (isAborted(options.signal))
|
|
135
|
+
return err(cancelled());
|
|
136
|
+
const uniqueRelationships = deduplicateRelationships(relationships);
|
|
137
|
+
const uniqueFailures = [...parseFailures]
|
|
138
|
+
.sort((left, right) => {
|
|
139
|
+
const byPath = byCodePoint(left.path, right.path);
|
|
140
|
+
if (byPath !== 0)
|
|
141
|
+
return byPath;
|
|
142
|
+
return byCodePoint(left.parser, right.parser);
|
|
143
|
+
})
|
|
144
|
+
.filter((value, index, array) => {
|
|
145
|
+
if (index === 0)
|
|
146
|
+
return true;
|
|
147
|
+
const previous = array[index - 1];
|
|
148
|
+
if (previous === undefined)
|
|
149
|
+
return true;
|
|
150
|
+
return value.path !== previous.path || value.parser !== previous.parser;
|
|
151
|
+
});
|
|
152
|
+
const sortedEntries = [...entries].sort((left, right) => byCodePoint(left.path, right.path));
|
|
153
|
+
const sortedExclusions = sortAndLimit(exclusions, limits.maxEntries);
|
|
154
|
+
const sortedLimitations = [...limitations].sort(byCodePoint);
|
|
155
|
+
const input = {
|
|
156
|
+
schema: "HistoricalSourceGraph/v1",
|
|
157
|
+
authority: "historical-reference",
|
|
158
|
+
root_alias: "$REFERENCE_ROOT",
|
|
159
|
+
inventory_state: deriveInventoryState({
|
|
160
|
+
entries: sortedEntries,
|
|
161
|
+
relationships: uniqueRelationships,
|
|
162
|
+
parse_failures: uniqueFailures,
|
|
163
|
+
exclusions: sortedExclusions,
|
|
164
|
+
limitations: sortedLimitations,
|
|
165
|
+
}),
|
|
166
|
+
entries: sortedEntries,
|
|
167
|
+
relationships: uniqueRelationships.sort((left, right) => byCodePoint(relationshipKey(left), relationshipKey(right))),
|
|
168
|
+
parse_failures: uniqueFailures,
|
|
169
|
+
exclusions: sortedExclusions,
|
|
170
|
+
languages: deriveLanguages(sortedEntries),
|
|
171
|
+
manifests: deriveManifests(sortedEntries),
|
|
172
|
+
vcs,
|
|
173
|
+
provenance: buildProvenance(options),
|
|
174
|
+
limitations: sortedLimitations,
|
|
175
|
+
};
|
|
176
|
+
try {
|
|
177
|
+
return ok(createHistoricalSourceGraph(input));
|
|
178
|
+
}
|
|
179
|
+
catch (error) {
|
|
180
|
+
return err(failure("parse", error instanceof Error ? error.message : "Graph failed validation"));
|
|
181
|
+
}
|
|
182
|
+
};
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { posix } from "node:path";
|
|
3
|
+
import { classifyReferenceSourcePath, detectReferenceSourceLanguage, } from "../domain/referenceSourceClassification.js";
|
|
4
|
+
import { parseReferenceSourceImports } from "../domain/referenceSourceImportParsing.js";
|
|
5
|
+
import { PARSEABLE_REFERENCE_SOURCE_LANGUAGES } from "./ReferenceSourceImportTypes.js";
|
|
6
|
+
const hashBytes = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
7
|
+
const failedEntry = (entry) => {
|
|
8
|
+
const classifications = classifyReferenceSourcePath(entry.path);
|
|
9
|
+
if (entry.kind === "directory")
|
|
10
|
+
return {
|
|
11
|
+
path: entry.path,
|
|
12
|
+
kind: "directory",
|
|
13
|
+
classifications,
|
|
14
|
+
tree_state: entry.code === "limit" ? "partial" : "unreadable",
|
|
15
|
+
limitations: [entry.message],
|
|
16
|
+
};
|
|
17
|
+
if (entry.kind === "symlink")
|
|
18
|
+
return {
|
|
19
|
+
path: entry.path,
|
|
20
|
+
kind: "symlink",
|
|
21
|
+
target: "<unreadable>",
|
|
22
|
+
target_state: "unreadable",
|
|
23
|
+
classifications,
|
|
24
|
+
limitations: [entry.message],
|
|
25
|
+
};
|
|
26
|
+
return {
|
|
27
|
+
path: entry.path,
|
|
28
|
+
kind: "file",
|
|
29
|
+
sha256: null,
|
|
30
|
+
size: entry.size ?? null,
|
|
31
|
+
language: null,
|
|
32
|
+
classifications: entry.kind === "file" ? classifications : ["unknown"],
|
|
33
|
+
content_state: entry.code === "limit" ? "too-large" : "unreadable",
|
|
34
|
+
limitations: [entry.message],
|
|
35
|
+
};
|
|
36
|
+
};
|
|
37
|
+
const resolveInternalSpecifier = (fromPath, specifier, filePaths) => {
|
|
38
|
+
const normalized = posix.normalize(posix.join(posix.dirname(fromPath), specifier));
|
|
39
|
+
if (normalized.startsWith("../") || normalized === "..")
|
|
40
|
+
return { to: normalized, resolution: "unresolved" };
|
|
41
|
+
const suffixes = ["", ".ts", ".js", ".mjs", ".cjs", ".tsx", ".jsx"];
|
|
42
|
+
const candidates = suffixes.flatMap((suffix) => [
|
|
43
|
+
`${normalized}${suffix}`,
|
|
44
|
+
`${normalized}/index${suffix}`,
|
|
45
|
+
]);
|
|
46
|
+
const match = candidates.find((candidate) => filePaths.has(candidate));
|
|
47
|
+
return match === undefined
|
|
48
|
+
? { to: normalized, resolution: "unresolved" }
|
|
49
|
+
: { to: match, resolution: "internal" };
|
|
50
|
+
};
|
|
51
|
+
const appendFile = (entry, filePaths, output) => {
|
|
52
|
+
const language = detectReferenceSourceLanguage(entry.path);
|
|
53
|
+
output.entries.push({
|
|
54
|
+
path: entry.path,
|
|
55
|
+
kind: "file",
|
|
56
|
+
sha256: hashBytes(entry.bytes),
|
|
57
|
+
size: entry.size,
|
|
58
|
+
language,
|
|
59
|
+
classifications: classifyReferenceSourcePath(entry.path),
|
|
60
|
+
content_state: "hashed",
|
|
61
|
+
limitations: [],
|
|
62
|
+
});
|
|
63
|
+
if (language === null || !PARSEABLE_REFERENCE_SOURCE_LANGUAGES.has(language))
|
|
64
|
+
return;
|
|
65
|
+
const parsed = parseReferenceSourceImports(entry.path, entry.bytes, language);
|
|
66
|
+
for (const relationship of parsed.relationships) {
|
|
67
|
+
if (relationship.resolution !== "internal") {
|
|
68
|
+
output.relationships.push(relationship);
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
const resolved = resolveInternalSpecifier(relationship.from_path, relationship.to, filePaths);
|
|
72
|
+
output.relationships.push({
|
|
73
|
+
...relationship,
|
|
74
|
+
to: resolved.to,
|
|
75
|
+
resolution: resolved.resolution,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
output.parseFailures.push(...parsed.parse_failures);
|
|
79
|
+
};
|
|
80
|
+
const appendReadEntry = (entry, filePaths, output) => {
|
|
81
|
+
if (entry.kind === "file") {
|
|
82
|
+
appendFile(entry, filePaths, output);
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
const classifications = classifyReferenceSourcePath(entry.path);
|
|
86
|
+
if (entry.kind === "directory") {
|
|
87
|
+
output.entries.push({
|
|
88
|
+
path: entry.path,
|
|
89
|
+
kind: "directory",
|
|
90
|
+
classifications,
|
|
91
|
+
tree_state: "enumerated",
|
|
92
|
+
limitations: [],
|
|
93
|
+
});
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
output.entries.push({
|
|
97
|
+
path: entry.path,
|
|
98
|
+
kind: "symlink",
|
|
99
|
+
target: entry.target,
|
|
100
|
+
target_state: entry.targetState,
|
|
101
|
+
classifications,
|
|
102
|
+
limitations: [],
|
|
103
|
+
});
|
|
104
|
+
};
|
|
105
|
+
/** Convert bounded reader observations into graph entries and parsed edges. */
|
|
106
|
+
export const parseReferenceSourceEntries = (read, filePaths, signal) => {
|
|
107
|
+
const output = {
|
|
108
|
+
entries: [],
|
|
109
|
+
relationships: [],
|
|
110
|
+
parseFailures: [],
|
|
111
|
+
limitations: [...read.limitations],
|
|
112
|
+
};
|
|
113
|
+
for (const entry of read.entries) {
|
|
114
|
+
if (signal?.aborted === true)
|
|
115
|
+
break;
|
|
116
|
+
if (entry.status === "failed")
|
|
117
|
+
output.entries.push(failedEntry(entry));
|
|
118
|
+
else
|
|
119
|
+
appendReadEntry(entry, filePaths, output);
|
|
120
|
+
}
|
|
121
|
+
return output;
|
|
122
|
+
};
|