rea-agents 4.0.1 → 5.0.0

package/dist/firmware/FirmwareFiles.js

@@ -0,0 +1,190 @@
+import { createHash } from "node:crypto";
+import { constants, createReadStream, createWriteStream } from "node:fs";
+import { chmod, lstat, open, readdir, readlink, realpath, } from "node:fs/promises";
+import { isAbsolute, join, relative, resolve, sep } from "node:path";
+import { Transform } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { AnalysisCancelledError, AnalysisInputError, AnalysisOutputError, } from "../domain/analysisErrorCore.js";
+import { readBoundedFileBytes } from "../process/BoundedFileBytes.js";
+import { FIRMWARE_LIMITS } from "./FirmwareRelease.js";
+/** Hash a regular tool or extracted file without retaining its bytes in memory. */
+export const hashFirmwareFile = async (path, signal) => {
+    const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+    const hash = createHash("sha256");
+    try {
+        if (!(await handle.stat()).isFile())
+            throw new Error(`Not a regular file: ${path}`);
+        for await (const chunk of handle.createReadStream({ autoClose: false })) {
+            signal?.throwIfAborted();
+            hash.update(chunk);
+        }
+    }
+    finally {
+        await handle.close();
+    }
+    return hash.digest("hex");
+};
+/** Copy and identify the exact bytes consumed by firmware providers. */
+export const snapshotFirmware = async (request, root, signal) => {
+    const sourcePath = resolve(request.input.path);
+    const source = await realpath(sourcePath);
+    const handle = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW);
+    const fullPath = join(root, "firmware.bin");
+    const hash = createHash("sha256");
+    let size = 0;
+    try {
+        const stat = await handle.stat();
+        if (!stat.isFile() ||
+            stat.size === 0 ||
+            stat.size > FIRMWARE_LIMITS.inputBytes)
+            throw new AnalysisInputError(request.operation, undefined, [
+                {
+                    path: ["path"],
+                    reason: "out_of_range",
+                    message: `Firmware must be a nonempty regular file no larger than ${FIRMWARE_LIMITS.inputBytes} bytes`,
+                },
+            ]);
+        const meter = new Transform({
+            transform(chunk, _encoding, callback) {
+                size += chunk.length;
+                if (size > FIRMWARE_LIMITS.inputBytes) {
+                    callback(new Error("Firmware grew beyond the input byte budget"));
+                    return;
+                }
+                hash.update(chunk);
+                callback(null, chunk);
+            },
+        });
+        await pipeline(handle.createReadStream({ autoClose: false }), meter, createWriteStream(fullPath, { flags: "wx", mode: 0o600 }), { signal });
+    }
+    finally {
+        await handle.close();
+    }
+    await chmod(fullPath, 0o400);
+    const sha256 = hash.digest("hex");
+    const selected = request.operation === "extract_firmware" ? request.input.range : undefined;
+    const offset = selected?.offset ?? 0;
+    const length = selected?.length ?? size;
+    if (length === 0 || offset > size || length > size - offset)
+        throw new AnalysisInputError(request.operation, undefined, [
+            {
+                path: ["range"],
+                reason: "out_of_range",
+                message: `Selected range must be within the ${size}-byte firmware file`,
+            },
+        ]);
+    let inputPath = fullPath;
+    if (selected !== undefined) {
+        inputPath = join(root, "selected.bin");
+        await pipeline(createReadStream(fullPath, { start: offset, end: offset + length - 1 }), createWriteStream(inputPath, { flags: "wx", mode: 0o600 }), { signal });
+        await chmod(inputPath, 0o400);
+    }
+    return {
+        subject: { path: sourcePath, sha256, format: "file" },
+        size,
+        inputPath,
+        selection: {
+            offset,
+            length,
+            sha256: selected === undefined
+                ? sha256
+                : await hashFirmwareFile(inputPath, signal),
+        },
+    };
+};
+/** Read a complete bounded report; oversized data is never reported as complete. */
+export const readFirmwareReport = async (path, operation) => {
+    const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+    try {
+        if (!(await handle.stat()).isFile())
+            throw new AnalysisOutputError(operation, "Report is not a regular file");
+        const bytes = await readBoundedFileBytes(handle, FIRMWARE_LIMITS.reportBytes);
+        if (bytes === undefined)
+            throw new AnalysisOutputError(operation, `Report exceeded ${FIRMWARE_LIMITS.reportBytes} bytes`);
+        try {
+            const value = JSON.parse(bytes.toString("utf8"));
+            return value;
+        }
+        catch (cause) {
+            throw new AnalysisOutputError(operation, "Tool report is not valid complete JSON", { cause });
+        }
+    }
+    finally {
+        await handle.close();
+    }
+};
+/** Inventory private output without following links, enforcing aggregate staging budgets. */
+export const inventoryFirmwareOutput = async (root, maxBytes, maxEntries, operation, signal) => {
+    const entries = [];
+    const pending = [root];
+    let bytes = 0;
+    let count = 0;
+    while (pending.length > 0) {
+        if (signal?.aborted === true)
+            throw new AnalysisCancelledError(operation);
+        const directory = pending.pop();
+        if (directory === undefined)
+            break;
+        const directoryStat = await lstat(directory).catch((cause) => {
+            if (cause instanceof Error && "code" in cause && cause.code === "ENOENT")
+                return undefined;
+            throw cause;
+        });
+        if (directoryStat === undefined ||
+            !directoryStat.isDirectory() ||
+            directoryStat.isSymbolicLink())
+            continue;
+        const names = await readdir(directory).catch((cause) => {
+            if (cause instanceof Error && "code" in cause && cause.code === "ENOENT")
+                return [];
+            throw cause;
+        });
+        for (const name of names.sort()) {
+            count++;
+            if (count > maxEntries)
+                throw new AnalysisOutputError(operation, `Staging entry budget exceeded (${maxEntries})`);
+            const path = join(directory, name);
+            const stat = await lstat(path).catch((cause) => {
+                if (cause instanceof Error &&
+                    "code" in cause &&
+                    cause.code === "ENOENT")
+                    return undefined;
+                throw cause;
+            });
+            if (stat === undefined)
+                continue;
+            if (stat.isDirectory()) {
+                pending.push(path);
+                continue;
+            }
+            if (stat.isFile())
+                bytes += stat.size;
+            if (bytes > maxBytes)
+                throw new AnalysisOutputError(operation, `Staging byte budget exceeded (${maxBytes})`);
+            entries.push({
+                path,
+                relativePath: relative(root, path).split(sep).join("/"),
+                size: stat.size,
+                kind: stat.isSymbolicLink()
+                    ? "symlink"
+                    : stat.isFile()
+                        ? "file"
+                        : "special",
+                linkTarget: stat.isSymbolicLink() ? await readlink(path) : null,
+            });
+        }
+    }
+    return entries.sort((a, b) => a.relativePath.localeCompare(b.relativePath));
+};
+/** Bind producer paths to the selected input or private extraction tree. */
+export const firmwareLogicalPath = (path, inputPath, outputRoot) => {
+    if (path === inputPath)
+        return "$input";
+    const rel = relative(outputRoot, path);
+    if (!isAbsolute(path) ||
+        rel === ".." ||
+        rel.startsWith(`..${sep}`) ||
+        isAbsolute(rel))
+        throw new AnalysisOutputError("extract_firmware", `Provider reported a path outside its owned output: ${path}`);
+    return rel === "" ? "$output" : `$output/${rel.split(sep).join("/")}`;
+};

package/dist/firmware/FirmwareProvider.js

@@ -0,0 +1,264 @@
+import { join, isAbsolute } from "node:path";
+import { z } from "zod";
+import { createAnalysisExecution, } from "../application/AnalysisProvider.js";
+import { firmwareResultSchemas, } from "../domain/firmware/firmwareAnalysis.js";
+import { AnalysisError } from "../domain/analysisErrorBase.js";
+import { AnalysisCancelledError, AnalysisCapabilityUnavailableError, AnalysisInputError, AnalysisOutputError, AnalysisTimeoutError, } from "../domain/analysisErrorCore.js";
+import { ProviderAdapterError } from "../domain/providerAdapterError.js";
+import { ProviderCleanupError } from "../domain/providerCleanupError.js";
+import { err, ok } from "../domain/result.js";
+import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js";
+import { publishFirmwareExtraction } from "./FirmwarePublication.js";
+import { FIRMWARE_LIMITS, BINWALK_PROVIDER_IDENTITY, UNBLOB_PROVIDER_IDENTITY, } from "./FirmwareRelease.js";
+import { admitFirmwareVersion } from "./FirmwareVersion.js";
+import { resolveFirmwareCommand, runFirmwareCommand, readFirmwareCommandReport, } from "./FirmwareCommand.js";
+import { inventoryFirmwareOutput, snapshotFirmware } from "./FirmwareFiles.js";
+import { normalizeBinwalkReport, normalizeUnblobReport, } from "./FirmwareReports.js";
+const waitForPredecessor = (predecessor, operation, signal) => new Promise((resolve, reject) => {
+    const abort = () => reject(new AnalysisCancelledError(operation));
+    signal?.addEventListener("abort", abort, { once: true });
+    if (signal?.aborted === true)
+        abort();
+    void predecessor.then(() => {
+        signal?.removeEventListener("abort", abort);
+        resolve();
+    });
+});
+const limitations = [
+    "Signature matches and reported sizes do not establish partition validity, encryption or runtime mappings. Binwalk may infer unknown lengths before emitting its report.",
+    "Only Linux is supported by this integration. One worker, inherited per-process address-space/file-size/CPU limits and a deadline constrain execution; staging byte and entry budgets are polled and may transiently overshoot. Limits are not aggregate process-memory or filesystem quotas.",
+    "Unblob's default sandbox and skip rules remain enabled. Missing external extractors, unknown chunks, encrypted chunks and depth limits yield partial coverage. Extracted code is never executed; only regular files are published, without executable permission bits.",
+    "The actual launcher executable is hashed. Interpreter, shared libraries and installed Python package dependencies are not independently attested; source revision remains unknown for caller-supplied builds.",
+];
+/** Queued bring-your-own firmware adapters with private snapshots and owned cleanup. */
+export class FirmwareProvider {
+    environment;
+    launcher;
+    #tail = Promise.resolve();
+    #cleanupFailure;
+    constructor(environment = process.env, launcher) {
+        this.environment = environment;
+        this.launcher = launcher;
+    }
+    /** Inspect regions or explicitly extract selected bytes without changing the source. */
+    async execute(request, options) {
+        const predecessor = this.#tail;
+        let release = () => { };
+        this.#tail = new Promise((resolve) => {
+            release = resolve;
+        });
+        // Cancel the caller promptly without letting a later request bypass its predecessor.
+        try {
+            await waitForPredecessor(predecessor, request.operation, options?.signal);
+        }
+        catch (cause) {
+            void predecessor.then(release);
+            return err(cause instanceof AnalysisError
+                ? cause
+                : new AnalysisCancelledError(request.operation));
+        }
+        try {
+            return await this.#execute(request, options);
+        }
+        finally {
+            release();
+        }
+    }
+    async #execute(request, options) {
+        let root;
+        const engineName = request.operation === "inspect_firmware_regions" ? "binwalk" : "unblob";
+        const deadline = AbortSignal.timeout(FIRMWARE_LIMITS.timeoutMs);
+        const signal = options?.signal === undefined
+            ? deadline
+            : AbortSignal.any([options.signal, deadline]);
+        let outcome;
+        try {
+            if (this.#cleanupFailure !== undefined)
+                return err(this.#cleanupFailure);
+            if (signal.aborted)
+                throw new AnalysisCancelledError(request.operation);
+            const command = await resolveFirmwareCommand(this.environment, engineName, request.operation);
+            root = await PrivateRuntimeRoot.create({ prefix: "rea-firmware-" });
+            const target = await snapshotFirmware(request, root.path, signal);
+            const run = (args, outputBudget) => runFirmwareCommand({
+                engine: engineName,
+                operation: request.operation,
+                ...command,
+                args,
+                cwd: root?.path ?? "",
+                environment: this.environment,
+                signal,
+                ...(this.launcher === undefined ? {} : { launcher: this.launcher }),
+                ...(outputBudget === undefined
+                    ? {}
+                    : { outputBudget, acceptReportedExtractionFailure: true }),
+            });
+            const versionRun = await run(["--version"]);
+            const version = versionRun.stdout.text.trim();
+            if (versionRun.stdout.bytes > 1024)
+                throw new AnalysisCapabilityUnavailableError(engineName, request.operation, `Tool version banner is ${String(versionRun.stdout.bytes)} bytes; expected a short ${engineName} version line.`);
+            const admitted = admitFirmwareVersion(engineName, version);
+            if (admitted.status === "unsupported" || admitted.status === "unresolved")
+                throw new AnalysisCapabilityUnavailableError(engineName, request.operation, admitted.message);
+            const engine = {
+                name: engineName,
+                version: admitted.version,
+                executable_path: command.command,
+                executable_sha256: command.sha256,
+                source_revision: null,
+                worker_count: 1,
+            };
+            const reportPath = join(root.path, "report.json");
+            let result;
+            let raw;
+            if (request.operation === "inspect_firmware_regions") {
+                const processRun = await run([
+                    "--quiet",
+                    "--search-all",
+                    "--threads",
+                    "1",
+                    "--log",
+                    reportPath,
+                    target.inputPath,
+                ]);
+                const report = await readFirmwareCommandReport({
+                    path: reportPath,
+                    engine: engineName,
+                    operation: request.operation,
+                    execution: processRun,
+                });
+                result = firmwareResultSchemas.inspect_firmware_regions.parse({
+                    engine,
+                    input_size: target.size,
+                    regions: normalizeBinwalkReport(report, target.inputPath, target.size),
+                    coverage: "signature_scan",
+                });
+                raw = { report, execution: processRun, version_execution: versionRun };
+            }
+            else {
+                if (!isAbsolute(request.input.output_directory))
+                    throw new AnalysisInputError(request.operation, undefined, [
+                        {
+                            path: ["output_directory"],
+                            reason: "invalid_format",
+                            message: "Extraction output directory must be absolute",
+                        },
+                    ]);
+                const outputRoot = join(root.path, "extracted");
+                const processRun = await run([
+                    "--process-num",
+                    "1",
+                    "--depth",
+                    String(request.input.max_depth),
+                    "--randomness-depth",
+                    "0",
+                    "--report",
+                    reportPath,
+                    "--log",
+                    join(root.path, "unblob.log"),
+                    "--extract-dir",
+                    outputRoot,
+                    target.inputPath,
+                ], {
+                    root: outputRoot,
+                    bytes: request.input.max_output_bytes,
+                    entries: request.input.max_output_files,
+                });
+                const report = await readFirmwareCommandReport({
+                    path: reportPath,
+                    engine: engineName,
+                    operation: request.operation,
+                    execution: processRun,
+                });
+                const normalized = normalizeUnblobReport(report, {
+                    inputPath: target.inputPath,
+                    outputRoot,
+                    ...target.selection,
+                    maxDepth: request.input.max_depth,
+                });
+                if (processRun.exit_code === 1 && normalized.diagnostics.length === 0)
+                    throw new AnalysisOutputError(request.operation, "Unblob failed without a reported extraction diagnostic");
+                const entries = await inventoryFirmwareOutput(outputRoot, request.input.max_output_bytes, request.input.max_output_files, request.operation, signal);
+                result = await publishFirmwareExtraction({
+                    outputDirectory: request.input.output_directory,
+                    entries,
+                    normalized,
+                    selection: target.selection,
+                    engine,
+                    signal,
+                });
+                raw = { report, execution: processRun, version_execution: versionRun };
+            }
+            const catalog = engineName === "binwalk"
+                ? BINWALK_PROVIDER_IDENTITY
+                : UNBLOB_PROVIDER_IDENTITY;
+            outcome = ok(createAnalysisExecution(result, { ...catalog, version: admitted.version }, {
+                subject: target.subject,
+                rawResult: raw,
+                limitations: [
+                    ...limitations,
+                    ...(admitted.status === "compatible"
+                        ? [admitted.limitation]
+                        : []),
+                ],
+                locations: [
+                    {
+                        kind: "file-offset-range",
+                        start: target.selection.offset,
+                        end: target.selection.offset + target.selection.length,
+                    },
+                ],
+            }));
+        }
+        catch (cause) {
+            outcome = err(cause instanceof ProviderCleanupError
+                ? cause
+                : options?.signal?.aborted === true
+                    ? new AnalysisCancelledError(request.operation)
+                    : deadline.aborted
+                        ? new AnalysisTimeoutError(request.operation, FIRMWARE_LIMITS.timeoutMs)
+                        : cause instanceof AnalysisError
+                            ? cause
+                            : cause instanceof z.ZodError
+                                ? new AnalysisOutputError(request.operation, "Tool report violated the pinned producer protocol", { cause })
+                                : new ProviderAdapterError(engineName, request.operation, {
+                                    cause,
+                                    diagnostics: {
+                                        reason: cause instanceof Error
+                                            ? cause.message
+                                            : String(cause),
+                                    },
+                                }));
+            if (!outcome.ok && outcome.error.cleanupIncomplete) {
+                this.#cleanupFailure =
+                    root === undefined ||
+                        outcome.error.cleanupResources.includes(root.path)
+                        ? outcome.error
+                        : new ProviderCleanupError(engineName, [...outcome.error.cleanupResources, root.path], outcome.error instanceof ProviderAdapterError
+                            ? (outcome.error.diagnostics ?? {
+                                reason: outcome.error.message,
+                            })
+                            : { reason: outcome.error.message }, { cause: outcome.error });
+            }
+        }
+        // Never remove a workspace while process ownership remains uncertain.
+        if (this.#cleanupFailure !== undefined)
+            return err(this.#cleanupFailure);
+        try {
+            await root?.close();
+        }
+        catch (cause) {
+            this.#cleanupFailure = new ProviderCleanupError(engineName, [
+                root?.path ?? "unknown",
+                ...(outcome.ok && request.operation === "extract_firmware"
+                    ? [request.input.output_directory]
+                    : []),
+            ], {
+                reason: cause instanceof Error ? cause.message : String(cause),
+                previous_result: outcome.ok ? outcome.value.result : null,
+            });
+            return err(this.#cleanupFailure);
+        }
+        return outcome;
+    }
+}

package/dist/firmware/FirmwarePublication.js

@@ -0,0 +1,84 @@
+import { createReadStream } from "node:fs";
+import { join } from "node:path";
+import { SafeOutputTree } from "../artifacts/SafeOutputTree.js";
+import { AnalysisOutputError } from "../domain/analysisErrorCore.js";
+import { ProviderCleanupError } from "../domain/providerCleanupError.js";
+import { firmwareResultSchemas } from "../domain/firmware/firmwareAnalysis.js";
+import { hashFirmwareFile } from "./FirmwareFiles.js";
+/** Publish verified regular files, preserving links as metadata and rolling back failed writes. */
+export const publishFirmwareExtraction = async (context) => {
+    let tree;
+    try {
+        const materialized = new Set(context.entries
+            .filter((entry) => entry.kind === "file")
+            .map((entry) => `$output/${entry.relativePath}`));
+        for (const path of context.normalized.files.keys()) {
+            if (path !== "$input" && !materialized.has(path))
+                throw new AnalysisOutputError("extract_firmware", `Reported regular file is absent from materialized output: ${path}`);
+        }
+        const files = [];
+        const unpublished = [];
+        tree = await SafeOutputTree.create(context.outputDirectory);
+        for (const entry of context.entries) {
+            if (entry.kind !== "file") {
+                unpublished.push({
+                    relative_path: entry.relativePath,
+                    kind: entry.kind,
+                    link_target: entry.linkTarget,
+                });
+                continue;
+            }
+            const logicalPath = `$output/${entry.relativePath}`;
+            const reported = context.normalized.files.get(logicalPath);
+            const sha256 = await hashFirmwareFile(entry.path, context.signal);
+            if ((reported?.sha256 !== null &&
+                reported?.sha256 !== undefined &&
+                reported.sha256 !== sha256) ||
+                (context.normalized.sizes.has(logicalPath) &&
+                    context.normalized.sizes.get(logicalPath) !== entry.size))
+                throw new AnalysisOutputError("extract_firmware", `Extracted bytes disagree with provider report: ${entry.relativePath}`);
+            await tree.write(entry.relativePath, createReadStream(entry.path), sha256, context.signal);
+            files.push({
+                relative_path: entry.relativePath,
+                path: join(tree.outputRoot, entry.relativePath),
+                size: entry.size,
+                sha256,
+                reported_sha256: reported?.sha256 ?? null,
+                mime_type: reported?.mimeType ?? null,
+                analysis_depth: reported?.depth ?? null,
+                runtime_address: null,
+                original_file_range: null,
+            });
+        }
+        const result = firmwareResultSchemas.extract_firmware.parse({
+            engine: context.engine,
+            selection: context.selection,
+            output_directory: tree.outputRoot,
+            coverage: unpublished.length > 0 ? "partial" : context.normalized.coverage,
+            files,
+            unpublished_entries: unpublished,
+            chunks: context.normalized.chunks,
+            derivations: context.normalized.derivations,
+            depth_limited_paths: context.normalized.depth_limited_paths,
+            diagnostics: context.normalized.diagnostics,
+            published_bytes: files.reduce((sum, file) => sum + file.size, 0),
+        });
+        context.signal.throwIfAborted();
+        await tree.commit();
+        return result;
+    }
+    catch (cause) {
+        try {
+            await tree?.rollback();
+        }
+        catch (cleanupCause) {
+            throw new ProviderCleanupError("unblob", [context.outputDirectory], {
+                reason: cleanupCause instanceof Error
+                    ? cleanupCause.message
+                    : String(cleanupCause),
+                previous_error: cause instanceof Error ? cause.message : String(cause),
+            }, { cause: cleanupCause });
+        }
+        throw cause;
+    }
+};

package/dist/firmware/FirmwareRelease.js

@@ -0,0 +1,32 @@
+/** Audited upstream sources; installed tool bytes are fingerprinted separately. */
+export const FIRMWARE_RELEASES = {
+    binwalk: {
+        version: "3.1.0",
+        revision: "4fdab3d464d97b68e0af9088df3f9e2e1545b21c",
+        repository: "https://github.com/ReFirmLabs/binwalk",
+    },
+    unblob: {
+        version: "26.6.4",
+        revision: "1fcc7a0a584a70a96c31f5a276c20944d199a089",
+        repository: "https://github.com/onekey-sec/unblob",
+    },
+};
+/** Catalog identity for the verified Binwalk release. Evidence replaces version with the observed banner. */
+export const BINWALK_PROVIDER_IDENTITY = {
+    id: "binwalk",
+    name: "Binwalk",
+    version: FIRMWARE_RELEASES.binwalk.version,
+};
+/** Catalog identity for explicit firmware extraction. */
+export const UNBLOB_PROVIDER_IDENTITY = {
+    id: "unblob",
+    name: "Unblob",
+    version: FIRMWARE_RELEASES.unblob.version,
+};
+/** Resource policy for this Linux integration, not a claim of aggregate containment. */
+export const FIRMWARE_LIMITS = {
+    inputBytes: 128 * 1024 * 1024,
+    reportBytes: 8 * 1024 * 1024,
+    addressSpaceBytes: 1024 * 1024 * 1024,
+    timeoutMs: 120_000,
+};