rea-agents 6.0.0 → 6.2.0

package/dist/evaluation/AgentEvaluationFixtures.js

@@ -0,0 +1,250 @@
+import { createHash } from "node:crypto";
+import { cp, mkdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { createPackage } from "@electron/asar";
+import { javascriptApplicationAnalysisResultSchema } from "../domain/javascript/javascriptApplicationAnalysis.js";
+import { javaScriptExportShapeComparisonResultSchema } from "../domain/javascript/javascriptExportShapeComparisonSchemas.js";
+const desktopFixtureFiles = {
+    "package.json": `${JSON.stringify({ name: "desktop-fixture", version: "1.0.0", main: "main.js" }, null, 2)}\n`,
+    "main.js": 'const { BrowserWindow, ipcMain } = require("electron");\nnew BrowserWindow({ webPreferences: { preload: require("node:path").join(__dirname, "preload.js"), contextIsolation: true, sandbox: true } });\nipcMain.handle("profile:read", (_event, id) => ({ id }));\n',
+    "preload.js": 'const { contextBridge, ipcRenderer } = require("electron");\ncontextBridge.exposeInMainWorld("profileApi", { read: (id) => ipcRenderer.invoke("profile:read", id) });\n',
+    "renderer/app.js": 'globalThis.profileApi.read("fixture");\n',
+};
+/** Materialize the exact packaged and source-owned parser targets used by the runner. */
+export const createAgentEvaluationFixtures = async (root, repositoryRoot) => {
+    const desktop = join(root, "desktop-app");
+    const javascript = join(root, "desktop-app.asar");
+    const javascriptShapeLeft = join(root, "parser-v1");
+    const javascriptShapeRight = join(root, "parser-v2");
+    await Promise.all([
+        mkdir(join(desktop, "renderer"), { recursive: true }),
+        mkdir(javascriptShapeLeft, { recursive: true }),
+        mkdir(javascriptShapeRight, { recursive: true }),
+    ]);
+    await Promise.all([
+        ...Object.entries(desktopFixtureFiles).map(([path, source]) => writeFile(join(desktop, path), source)),
+        cp(join(repositoryRoot, "tests/fixtures/replay/parser.mjs"), join(javascriptShapeLeft, "parser.mjs")),
+        cp(join(repositoryRoot, "tests/fixtures/replay/parser-v2.mjs"), join(javascriptShapeRight, "parser.mjs")),
+    ]);
+    await createPackage(desktop, javascript);
+    return {
+        javascript,
+        javascriptSha256: createHash("sha256")
+            .update(await readFile(javascript))
+            .digest("hex"),
+        javascriptShapeLeft,
+        javascriptShapeRight,
+    };
+};
+const singleObservation = (value, kind, include = () => true) => {
+    const result = javascriptApplicationAnalysisResultSchema.safeParse(value);
+    if (!result.success)
+        return undefined;
+    const candidates = result.data.graph.nodes
+        .filter((node) => node.kind === kind)
+        .flatMap((node) => node.observations)
+        .filter(include);
+    return candidates.length === 1 ? candidates[0] : undefined;
+};
+const observationSource = (observation) => {
+    const location = observation.evidence.location;
+    return location.available && location.value.kind === "source-range"
+        ? location.value.source
+        : undefined;
+};
+const rendererApi = (value) => {
+    const observation = singleObservation(value, "context-bridge-api");
+    if (observation === undefined)
+        return undefined;
+    const { api_key: key, members, world } = observation.properties;
+    const source = observationSource(observation);
+    if (key === undefined ||
+        members === undefined ||
+        world === undefined ||
+        source === undefined)
+        return undefined;
+    return { key, members, world, source };
+};
+const ipcOperation = (side) => (value) => {
+    const observation = singleObservation(value, "ipc-channel", (candidate) => candidate.properties.side === side);
+    if (observation === undefined)
+        return undefined;
+    const { channel, operation } = observation.properties;
+    const source = observationSource(observation);
+    if (channel === undefined ||
+        operation === undefined ||
+        source === undefined)
+        return undefined;
+    return { channel, operation, source };
+};
+const preload = (value) => {
+    const observation = singleObservation(value, "electron-preload", (candidate) => candidate.properties.mechanism === "BrowserWindow:webPreferences.preload");
+    const path = observation?.properties.resolved_path;
+    const resolutionStatus = observation?.properties.resolution_status;
+    if (path === undefined || resolutionStatus === undefined)
+        return undefined;
+    return { path, resolution_status: resolutionStatus };
+};
+const headingChange = (value) => {
+    const result = javaScriptExportShapeComparisonResultSchema.safeParse(value);
+    if (!result.success)
+        return undefined;
+    const changes = result.data.changes.filter((change) => change.path === "/depth");
+    const change = changes.length === 1 ? changes[0] : undefined;
+    if (change === undefined)
+        return undefined;
+    return {
+        status: change.status,
+        path: change.path,
+        discriminant: change.discriminant,
+        left: change.left,
+        right: change.right,
+    };
+};
+const comparisonCoverage = (value) => {
+    const result = javaScriptExportShapeComparisonResultSchema.safeParse(value);
+    if (!result.success)
+        return undefined;
+    return {
+        status: result.data.coverage.status,
+        paired_variants: result.data.coverage.paired_variants,
+        ...result.data.summary,
+    };
+};
+const runtimeSemantics = (value) => {
+    const result = javaScriptExportShapeComparisonResultSchema.safeParse(value);
+    return result.success &&
+        result.data.limitations.includes("Return shapes are inferred from inert syntax and do not prove runtime behavior.")
+        ? "not_established"
+        : undefined;
+};
+const asarClaims = (targets) => {
+    const source = {
+        operation: "analyze_javascript_application",
+        authority: "shipped-artifact",
+        confidence: "derived",
+        subject: { path: targets.javascript, sha256: targets.javascriptSha256 },
+    };
+    return [
+        {
+            id: "renderer_api",
+            expectedValue: {
+                key: "profileApi",
+                members: ["read"],
+                world: "main",
+                source: "preload.js",
+            },
+            source: { ...source, select: rendererApi },
+        },
+        {
+            id: "renderer_ipc",
+            expectedValue: {
+                channel: "profile:read",
+                operation: "invoke",
+                source: "preload.js",
+            },
+            source: { ...source, select: ipcOperation("renderer") },
+        },
+        {
+            id: "main_ipc",
+            expectedValue: {
+                channel: "profile:read",
+                operation: "handle",
+                source: "main.js",
+            },
+            source: { ...source, select: ipcOperation("main") },
+        },
+        {
+            id: "preload",
+            expectedValue: { path: "preload.js", resolution_status: "resolved" },
+            source: { ...source, select: preload },
+        },
+    ];
+};
+const parserClaims = (targets) => {
+    const source = {
+        operation: "compare_javascript_export_shapes",
+        authority: "analyst-inference",
+        confidence: "inferred",
+        parameters: {
+            left_module_path: "parser.mjs",
+            left_export_name: "default",
+            right_module_path: "parser.mjs",
+            right_export_name: "default",
+        },
+        linkedSubjects: [
+            {
+                operation: "analyze_javascript_application",
+                path: targets.javascriptShapeLeft,
+            },
+            {
+                operation: "analyze_javascript_application",
+                path: targets.javascriptShapeRight,
+            },
+        ],
+    };
+    return [
+        {
+            id: "heading_change",
+            expectedValue: {
+                status: "added",
+                path: "/depth",
+                discriminant: { path: "/type", value: "heading" },
+                left: { availability: "absent" },
+                right: { availability: "literal", value: 1 },
+            },
+            source: { ...source, select: headingChange },
+        },
+        {
+            id: "comparison_coverage",
+            expectedValue: {
+                status: "complete-within-inputs",
+                paired_variants: 3,
+                added: 1,
+                removed: 0,
+                changed: 0,
+                unknown: 0,
+            },
+            source: { ...source, select: comparisonCoverage },
+        },
+        {
+            id: "runtime_semantics",
+            expectedValue: "not_established",
+            source: { ...source, select: runtimeSemantics },
+        },
+    ];
+};
+/** Return the closed known-answer rubric, or undefined for routing-only scenarios. */
+export const agentFixtureClaims = (scenarioId, targets) => {
+    if (scenarioId === "asar")
+        return asarClaims(targets);
+    if (scenarioId === "javascript-export-shape")
+        return parserClaims(targets);
+    return undefined;
+};
+const answerValueDescriptions = {
+    asar: [
+        'renderer_api: {"key": string, "members": string[], "world": string, "source": string}, describing the exposed bridge API, its members, world, and source file.',
+        'renderer_ipc: {"channel": string, "operation": string, "source": string}, describing the renderer-side IPC operation and its source file.',
+        'main_ipc: {"channel": string, "operation": string, "source": string}, describing the main-side IPC operation and its source file.',
+        'preload: {"path": string, "resolution_status": string}, describing the BrowserWindow-declared preload resolved within the shipped artifact.',
+    ],
+    "javascript-export-shape": [
+        'heading_change: {"status": string, "path": string, "discriminant": object or null, "left": object, "right": object}, copying the exact heading-change fields from the comparison result.',
+        'comparison_coverage: {"status": string, "paired_variants": number, "added": number, "removed": number, "changed": number, "unknown": number}, copying comparison coverage and summary counts.',
+        'runtime_semantics: "established" or "not_established", stating whether the cited comparison establishes runtime behavior.',
+    ],
+};
+/** Describe the strict answer contract without revealing any golden fixture values. */
+export const agentFixtureAnswerInstructions = (scenarioId) => {
+    const descriptions = answerValueDescriptions[scenarioId];
+    if (descriptions === undefined)
+        return "";
+    return [
+        "For this benchmark, return only one JSON object, without Markdown fences or surrounding prose.",
+        'The exact shape is {"claims": [{"id": string, "value": JSON value, "evidence_id": string, "confidence": string, "authority": string}]}.',
+        "Include exactly one claim per ID below, without additional fields or claims. Infer values only from REA tool results; do not read target files directly.",
+        "Each evidence_id must identify the successful REA result that supports its claim. Copy confidence and authority from that result's Evidence envelope.",
+        ...descriptions,
+    ].join("\n");
+};

package/dist/evaluation/AgentEvaluationReport.js

@@ -0,0 +1,28 @@
+/** Require configured facts to pass while preserving workflow and execution gates. */
+export const agentEvaluationPassed = (result) => result.exitCode === 0 &&
+    result.naturalUse &&
+    result.correctFirstTool &&
+    result.repeatedCallCount === 0 &&
+    result.inputValidationFailureCount === 0 &&
+    result.requiredToolSubsequenceMet &&
+    result.inputTokens > 0 &&
+    (result.factualCorrectness === "passed" ||
+        (result.factualCorrectness === "not_assessed" &&
+            result.answerHeuristicsMet &&
+            result.epistemicCuePresent));
+/** Keep assessed fixture outcomes and scenarios without truth oracles distinct. */
+export const summarizeFactualCorrectness = (results) => {
+    const passed = results.filter(({ factualCorrectness }) => factualCorrectness === "passed").length;
+    const failed = results.filter(({ factualCorrectness }) => factualCorrectness === "failed").length;
+    const status = failed > 0 ? "failed" : passed > 0 ? "passed" : "not_assessed";
+    return {
+        status,
+        scope: "configured_fixture_claims",
+        counts: {
+            assessed: passed + failed,
+            passed,
+            failed,
+            notAssessed: results.length - passed - failed,
+        },
+    };
+};

package/dist/evaluation/CodexAgentEval.js

@@ -1,3 +1,4 @@
+import { evaluateKnownAnswers, } from "./KnownAnswerEvaluation.js";
 import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
 const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
 const record = (value) => isRecord(value) ? value : undefined;
@@ -91,7 +92,7 @@ const consumeEvent = (state, value) => {
     state.cachedInputTokens = Math.max(state.cachedInputTokens, numberValue(usage?.cached_input_tokens));
     state.outputTokens = Math.max(state.outputTokens, numberValue(usage?.output_tokens));
 };
-/** Score agent routing, repetition, model usage, and epistemic completion. */
+/** Measure agent routing, repetition, model usage, and explicitly limited answer heuristics. */
 export const evaluateCodexEvents = (events, expectedFirstTool, options = {}) => {
     const state = {
         calls: [],
@@ -118,8 +119,9 @@ export const evaluateCodexEvents = (events, expectedFirstTool, options = {}) =>
     ];
     const finalCitesEvidence = evidenceIds.some((evidenceId) => finalMessage.includes(evidenceId));
     const normalizedFinalMessage = finalMessage.toLocaleLowerCase("en-US");
-    const contentCriteriaMet = (options.requiredAnswerTermGroups ?? []).every((terms) => terms.some((term) => normalizedFinalMessage.includes(term.toLocaleLowerCase("en-US"))));
-    const authorityHonesty = /\b(evidence|observed|inferred|unknown|unavailable|limitation|authority|not configured|could not|requires approval)\b/iu.test(finalMessage);
+    const answerTermCoverageMet = (options.requiredAnswerTermGroups ?? []).every((terms) => terms.some((term) => normalizedFinalMessage.includes(term.toLocaleLowerCase("en-US"))));
+    const epistemicCuePresent = /\b(evidence|observed|inferred|unknown|unavailable|limitation|authority|not configured|could not|requires approval)\b/iu.test(finalMessage);
+    const factualAssessment = evaluateKnownAnswers(events, finalMessage, options.fixtureClaims);
     return {
         naturalUse: reaCalls.length > 0,
         correctFirstTool: reaCalls[0]?.tool === expectedFirstTool,
@@ -134,15 +136,17 @@ export const evaluateCodexEvents = (events, expectedFirstTool, options = {}) =>
         finalMessage,
         evidenceIds,
         finalCitesEvidence,
-        contentCriteriaMet,
-        completionQuality: finalMessage.trim().length >= 80 &&
-            authorityHonesty &&
-            contentCriteriaMet &&
+        answerTermCoverageMet,
+        answerHeuristicsMet: finalMessage.trim().length >= 80 &&
+            epistemicCuePresent &&
+            answerTermCoverageMet &&
             requiredToolSubsequenceMet &&
             (options.forbidInputValidationFailures !== true ||
                 inputValidationFailureCount === 0) &&
             (options.requireEvidence !== true || finalCitesEvidence),
-        authorityHonesty,
+        epistemicCuePresent,
+        factualCorrectness: factualAssessment.status,
+        factualAssessment,
     };
 };
 const containsOrderedSubsequence = (values, required) => {

package/dist/evaluation/KnownAnswerEvaluation.js

@@ -0,0 +1,281 @@
+import { visit } from "jsonc-parser";
+import { z } from "zod";
+import { parseEvidence } from "../domain/evidence.js";
+import { jsonValueSchema } from "../domain/jsonValue.js";
+import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
+const authoritySchema = z.enum([
+    "shipped-artifact",
+    "controlled-replay",
+    "historical-reference",
+    "external-service",
+    "analyst-inference",
+]);
+const confidenceSchema = z.enum(["observed", "derived", "inferred"]);
+const subjectSchema = z.strictObject({
+    path: z.string().min(1),
+    sha256: z
+        .string()
+        .regex(/^[a-f0-9]{64}$/u)
+        .optional(),
+});
+const expectationSchema = z.strictObject({
+    id: z.string().min(1),
+    expectedValue: jsonValueSchema,
+    source: z.strictObject({
+        operation: z.string().min(1),
+        authority: authoritySchema,
+        confidence: confidenceSchema,
+        subject: subjectSchema.optional(),
+        parameters: z.record(z.string(), jsonValueSchema).optional(),
+        linkedSubjects: z
+            .array(subjectSchema.extend({ operation: z.string().min(1) }))
+            .optional(),
+        select: z.custom((value) => typeof value === "function"),
+    }),
+});
+const finalAnswerSchema = z.strictObject({
+    claims: z.array(z.strictObject({
+        id: z.string().min(1),
+        value: jsonValueSchema,
+        evidence_id: z.string().regex(/^ev_[a-f0-9]{64}$/u),
+        confidence: confidenceSchema,
+        authority: authoritySchema,
+    })),
+});
+const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+const record = (value) => isRecord(value) ? value : undefined;
+const canonicalValue = (value) => {
+    if (Array.isArray(value))
+        return value.map(canonicalValue);
+    if (value === null || typeof value !== "object")
+        return value;
+    return Object.fromEntries(Object.entries(value)
+        .sort(([left], [right]) => compareUnicodeCodePoints(left, right))
+        .map(([key, child]) => [key, canonicalValue(child)]));
+};
+const sameJson = (left, right) => {
+    const parsedLeft = jsonValueSchema.safeParse(left);
+    const parsedRight = jsonValueSchema.safeParse(right);
+    return (parsedLeft.success &&
+        parsedRight.success &&
+        JSON.stringify(canonicalValue(parsedLeft.data)) ===
+            JSON.stringify(canonicalValue(parsedRight.data)));
+};
+const parseClosedAnswer = (text) => {
+    const objects = [];
+    let invalid = false;
+    try {
+        visit(text, {
+            onObjectBegin: () => {
+                objects.push(new Set());
+            },
+            onObjectProperty: (key) => {
+                const keys = objects.at(-1);
+                if (keys === undefined || keys.has(key))
+                    invalid = true;
+                keys?.add(key);
+            },
+            onObjectEnd: () => {
+                objects.pop();
+            },
+            onError: () => {
+                invalid = true;
+            },
+        }, {
+            disallowComments: true,
+            allowTrailingComma: false,
+            allowEmptyContent: false,
+        });
+        if (invalid)
+            return undefined;
+        // The JSON visitor checked syntax and duplicate members before JSON.parse can fold them.
+        return JSON.parse(text);
+    }
+    catch (error) {
+        // Syntax and nesting failures are malformed final-answer input, not an evaluation crash.
+        if (error instanceof SyntaxError || error instanceof RangeError)
+            return undefined;
+        throw error;
+    }
+};
+const consistentAliases = (value, left, right) => value[left] === undefined ||
+    value[right] === undefined ||
+    value[left] === value[right];
+const completedCall = (event) => {
+    const value = record(event);
+    const item = value?.type === "item.completed"
+        ? record(value.item)
+        : value?.type === "mcp_tool_call" && value.status === "completed"
+            ? value
+            : undefined;
+    if (item?.type !== "mcp_tool_call")
+        return undefined;
+    const server = item.server ?? item.server_name;
+    if (typeof server !== "string" ||
+        server.toLowerCase() !== "rea" ||
+        (item.status !== undefined && item.status !== "completed") ||
+        (item.error !== undefined && item.error !== null))
+        return undefined;
+    if (!consistentAliases(item, "server", "server_name") ||
+        !consistentAliases(item, "tool", "name"))
+        return undefined;
+    return item;
+};
+const consistentJsonAliases = (value, left, right) => value[left] === undefined ||
+    value[left] === null ||
+    value[right] === undefined ||
+    value[right] === null ||
+    sameJson(value[left], value[right]);
+const evidenceFromCall = (item) => {
+    if (!consistentJsonAliases(item, "result", "output"))
+        return undefined;
+    const result = record(item.result ?? item.output);
+    if (result === undefined ||
+        result.isError === true ||
+        (result.error !== undefined && result.error !== null))
+        return undefined;
+    const structured = record(result.structuredContent ?? result.structured_content);
+    if (structured === undefined ||
+        (structured.error !== undefined && structured.error !== null))
+        return undefined;
+    if (result.structuredContent !== undefined &&
+        result.structured_content !== undefined &&
+        !sameJson(result.structuredContent, result.structured_content))
+        return undefined;
+    let evidence;
+    try {
+        evidence = parseEvidence(structured);
+    }
+    catch {
+        // Transcript output is untrusted; invalid Evidence cannot support a claim.
+        return undefined;
+    }
+    if ((item.tool ?? item.name) !== evidence.operation)
+        return undefined;
+    return evidence;
+};
+const collectDeliveredEvidence = (events) => {
+    const records = new Map();
+    const conflicts = new Set();
+    for (const event of events) {
+        const item = completedCall(event);
+        if (item === undefined)
+            continue;
+        const evidence = evidenceFromCall(item);
+        if (evidence === undefined)
+            continue;
+        const previous = records.get(evidence.evidence_id);
+        if (previous !== undefined && !sameJson(previous, evidence))
+            conflicts.add(evidence.evidence_id);
+        else
+            records.set(evidence.evidence_id, evidence);
+    }
+    return { records, conflicts };
+};
+const matchesSubject = (evidence, expected) => evidence.subject?.local_path === expected.path &&
+    (expected.sha256 === undefined ||
+        evidence.subject.digest.sha256 === expected.sha256);
+const matchesParameters = (evidence, expected) => expected === undefined ||
+    Object.entries(expected).every(([key, value]) => Object.hasOwn(evidence.parameters, key) &&
+        sameJson(evidence.parameters[key], value));
+const matchesLinks = (evidence, source, delivered) => (source.linkedSubjects ?? []).every((expected) => evidence.evidence_links.some((id) => {
+    const linked = delivered.records.get(id);
+    return (linked !== undefined &&
+        !delivered.conflicts.has(id) &&
+        linked.operation === expected.operation &&
+        matchesSubject(linked, expected));
+}));
+const assessClaim = (expected, claim, delivered) => {
+    const failures = [];
+    if (claim === undefined)
+        return { id: expected.id, status: "failed", failures: ["claim_missing"] };
+    if (!sameJson(claim.value, expected.expectedValue))
+        failures.push("value_mismatch");
+    if (claim.confidence !== expected.source.confidence)
+        failures.push("claim_confidence_mismatch");
+    if (claim.authority !== expected.source.authority)
+        failures.push("claim_authority_mismatch");
+    const evidence = delivered.records.get(claim.evidence_id);
+    if (evidence === undefined)
+        failures.push("unknown_evidence");
+    else if (delivered.conflicts.has(claim.evidence_id))
+        failures.push("conflicting_evidence");
+    else {
+        if (evidence.operation !== expected.source.operation)
+            failures.push("source_operation_mismatch");
+        if (evidence.confidence !== expected.source.confidence)
+            failures.push("source_confidence_mismatch");
+        if (evidence.authority !== expected.source.authority)
+            failures.push("source_authority_mismatch");
+        if (expected.source.subject !== undefined &&
+            !matchesSubject(evidence, expected.source.subject))
+            failures.push("source_subject_mismatch");
+        if (!matchesParameters(evidence, expected.source.parameters))
+            failures.push("source_parameters_mismatch");
+        if (!matchesLinks(evidence, expected.source, delivered))
+            failures.push("source_links_mismatch");
+        try {
+            if (!sameJson(expected.source.select(evidence.normalized_result), expected.expectedValue))
+                failures.push("source_assertion_mismatch");
+        }
+        catch {
+            // Producer-specific selectors may reject malformed representations; preserve a failed assessment.
+            failures.push("selector_failed");
+        }
+    }
+    return {
+        id: expected.id,
+        status: failures.length === 0 ? "passed" : "failed",
+        failures,
+    };
+};
+/** Grade closed answers only against configured fixture truth and delivered authenticated Evidence. */
+export const evaluateKnownAnswers = (events, finalMessage, expectations) => {
+    const scope = "configured_fixture_claims";
+    if (expectations === undefined)
+        return { scope, status: "not_assessed", manifestFailures: [], claims: [] };
+    const rubric = z.array(expectationSchema).min(1).safeParse(expectations);
+    if (!rubric.success)
+        return {
+            scope,
+            status: "failed",
+            manifestFailures: ["invalid_fixture_rubric"],
+            claims: [],
+        };
+    const expectedIds = new Set(expectations.map(({ id }) => id));
+    if (expectedIds.size !== expectations.length)
+        return {
+            scope,
+            status: "failed",
+            manifestFailures: ["duplicate_fixture_claim_ids"],
+            claims: [],
+        };
+    const manifest = finalAnswerSchema.safeParse(parseClosedAnswer(finalMessage));
+    if (!manifest.success)
+        return {
+            scope,
+            status: "failed",
+            manifestFailures: ["invalid_final_claim_manifest"],
+            claims: [],
+        };
+    const manifestFailures = [];
+    const submitted = new Map();
+    for (const claim of manifest.data.claims) {
+        if (submitted.has(claim.id))
+            manifestFailures.push(`duplicate_claim:${claim.id}`);
+        if (!expectedIds.has(claim.id))
+            manifestFailures.push(`unknown_claim:${claim.id}`);
+        submitted.set(claim.id, claim);
+    }
+    const delivered = collectDeliveredEvidence(events);
+    const claims = expectations.map((expected) => assessClaim(expected, submitted.get(expected.id), delivered));
+    return {
+        scope,
+        status: manifestFailures.length === 0 &&
+            claims.every(({ status }) => status === "passed")
+            ? "passed"
+            : "failed",
+        manifestFailures,
+        claims,
+    };
+};

package/dist/evm/EvmoleInterfaceProvider.js

@@ -21,6 +21,7 @@ import { EVMOLE_PROVIDER_IDENTITY, EVM_INTERFACE_LIMITS, EVM_FILE_SIZE_FAILURE_E
 import { decodeEvmBytecodeCarrier } from "./EvmBytecodeCarrier.js";
 import { selectEvmWorkerLimits, } from "./EvmWorkerLimits.js";
 const OPERATION = "inspect_evm_interface";
+const HOST_REQUIREMENT = "Initial real-verified offline EVM interface profile supports Linux x64 only.";
 const replySchema = z.discriminatedUnion("ok", [
     z.strictObject({
         ok: z.literal(true),
@@ -60,7 +61,7 @@ export class EvmoleInterfaceProvider {
             if (options?.signal?.aborted)
                 throw new AnalysisCancelledError(OPERATION);
             if (process.platform !== "linux" || process.arch !== "x64")
-                throw new AnalysisCapabilityUnavailableError(EVMOLE_PROVIDER_IDENTITY.id, OPERATION, "Initial real-verified offline EVM interface profile supports Linux x64 only.");
+                throw new AnalysisCapabilityUnavailableError(EVMOLE_PROVIDER_IDENTITY.id, OPERATION, HOST_REQUIREMENT, { userMessage: HOST_REQUIREMENT });
             try {
                 if (!isAbsolute(limiter))
                     throw new Error("REA_EVM_PRLIMIT_COMMAND must be an absolute executable path.");

package/dist/firmware/FirmwareCommand.js

@@ -9,6 +9,10 @@ import { spawnOwnedProviderProcess, ProviderProcessSupervisor, } from "../proces
 import { cleanupOwnedProcessGroup } from "../process/ProcessOwnership.js";
 import { FIRMWARE_LIMITS } from "./FirmwareRelease.js";
 import { hashFirmwareFile, inventoryFirmwareOutput, readFirmwareReport, } from "./FirmwareFiles.js";
+/** Refuse an unmet host or tool requirement, which another target cannot satisfy. */
+export const firmwareToolUnavailable = (engine, operation, reason) => new AnalysisCapabilityUnavailableError(engine, operation, reason, {
+    userMessage: reason,
+});
 /** Resolve explicitly supplied executable paths; never install or search for engines. */
 export const resolveFirmwareCommand = async (environment, engine, operation) => {
     const variable = engine === "binwalk" ? "REA_BINWALK_COMMAND" : "REA_UNBLOB_COMMAND";
@@ -16,9 +20,9 @@ export const resolveFirmwareCommand = async (environment, engine, operation) =>
     if (process.platform !== "linux" ||
         configured === undefined ||
         !isAbsolute(configured))
-        throw new AnalysisCapabilityUnavailableError(engine, operation, `This integration requires Linux and an absolute ${variable} path to the caller-supplied tool`);
+        throw firmwareToolUnavailable(engine, operation, `This integration requires Linux and an absolute ${variable} path to the caller-supplied tool`);
     const command = await realpath(configured).catch((cause) => {
-        throw new AnalysisCapabilityUnavailableError(engine, operation, `${variable} is unavailable: ${cause instanceof Error ? cause.message : String(cause)}`);
+        throw firmwareToolUnavailable(engine, operation, `${variable} is unavailable: ${cause instanceof Error ? cause.message : String(cause)}`);
     });
     let sha256;
     try {
@@ -28,18 +32,18 @@ export const resolveFirmwareCommand = async (environment, engine, operation) =>
         sha256 = await hashFirmwareFile(command);
     }
     catch (cause) {
-        throw new AnalysisCapabilityUnavailableError(engine, operation, `${variable} is not a readable executable regular file: ${command}: ${cause instanceof Error ? cause.message : String(cause)}`);
+        throw firmwareToolUnavailable(engine, operation, `${variable} is not a readable executable regular file: ${command}: ${cause instanceof Error ? cause.message : String(cause)}`);
     }
     const limiter = environment.REA_FIRMWARE_PRLIMIT_COMMAND ?? "/usr/bin/prlimit";
     if (!isAbsolute(limiter))
-        throw new AnalysisCapabilityUnavailableError(engine, operation, "REA_FIRMWARE_PRLIMIT_COMMAND must be absolute");
+        throw firmwareToolUnavailable(engine, operation, "REA_FIRMWARE_PRLIMIT_COMMAND must be absolute");
     try {
         await access(limiter, constants.X_OK);
         if (!(await stat(limiter)).isFile())
             throw new TypeError("Configured path is not a regular file");
     }
     catch (cause) {
-        throw new AnalysisCapabilityUnavailableError(engine, operation, `Resource limiter is not an executable regular file: ${limiter}: ${cause instanceof Error ? cause.message : String(cause)}; provide util-linux prlimit`);
+        throw firmwareToolUnavailable(engine, operation, `Resource limiter is not an executable regular file: ${limiter}: ${cause instanceof Error ? cause.message : String(cause)}; provide util-linux prlimit`);
     }
     return { command, limiter, sha256 };
 };