reverseport 0.0.0-stage → 1.2.0

package/dist/evaluation/AgentEvaluationFixtures.js

@@ -0,0 +1,250 @@
+import { createHash } from "node:crypto";
+import { cp, mkdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { createPackage } from "@electron/asar";
+import { javascriptApplicationAnalysisResultSchema } from "../domain/javascript/javascriptApplicationAnalysis.js";
+import { javaScriptExportShapeComparisonResultSchema } from "../domain/javascript/javascriptExportShapeComparisonSchemas.js";
+const desktopFixtureFiles = {
+    "package.json": `${JSON.stringify({ name: "desktop-fixture", version: "1.0.0", main: "main.js" }, null, 2)}\n`,
+    "main.js": 'const { BrowserWindow, ipcMain } = require("electron");\nnew BrowserWindow({ webPreferences: { preload: require("node:path").join(__dirname, "preload.js"), contextIsolation: true, sandbox: true } });\nipcMain.handle("profile:read", (_event, id) => ({ id }));\n',
+    "preload.js": 'const { contextBridge, ipcRenderer } = require("electron");\ncontextBridge.exposeInMainWorld("profileApi", { read: (id) => ipcRenderer.invoke("profile:read", id) });\n',
+    "renderer/app.js": 'globalThis.profileApi.read("fixture");\n',
+};
+/** Materialize the exact packaged and source-owned parser targets used by the runner. */
+export const createAgentEvaluationFixtures = async (root, repositoryRoot) => {
+    const desktop = join(root, "desktop-app");
+    const javascript = join(root, "desktop-app.asar");
+    const javascriptShapeLeft = join(root, "parser-v1");
+    const javascriptShapeRight = join(root, "parser-v2");
+    await Promise.all([
+        mkdir(join(desktop, "renderer"), { recursive: true }),
+        mkdir(javascriptShapeLeft, { recursive: true }),
+        mkdir(javascriptShapeRight, { recursive: true }),
+    ]);
+    await Promise.all([
+        ...Object.entries(desktopFixtureFiles).map(([path, source]) => writeFile(join(desktop, path), source)),
+        cp(join(repositoryRoot, "tests/fixtures/replay/parser.mjs"), join(javascriptShapeLeft, "parser.mjs")),
+        cp(join(repositoryRoot, "tests/fixtures/replay/parser-v2.mjs"), join(javascriptShapeRight, "parser.mjs")),
+    ]);
+    await createPackage(desktop, javascript);
+    return {
+        javascript,
+        javascriptSha256: createHash("sha256")
+            .update(await readFile(javascript))
+            .digest("hex"),
+        javascriptShapeLeft,
+        javascriptShapeRight,
+    };
+};
+const singleObservation = (value, kind, include = () => true) => {
+    const result = javascriptApplicationAnalysisResultSchema.safeParse(value);
+    if (!result.success)
+        return undefined;
+    const candidates = result.data.graph.nodes
+        .filter((node) => node.kind === kind)
+        .flatMap((node) => node.observations)
+        .filter(include);
+    return candidates.length === 1 ? candidates[0] : undefined;
+};
+const observationSource = (observation) => {
+    const location = observation.evidence.location;
+    return location.available && location.value.kind === "source-range"
+        ? location.value.source
+        : undefined;
+};
+const rendererApi = (value) => {
+    const observation = singleObservation(value, "context-bridge-api");
+    if (observation === undefined)
+        return undefined;
+    const { api_key: key, members, world } = observation.properties;
+    const source = observationSource(observation);
+    if (key === undefined ||
+        members === undefined ||
+        world === undefined ||
+        source === undefined)
+        return undefined;
+    return { key, members, world, source };
+};
+const ipcOperation = (side) => (value) => {
+    const observation = singleObservation(value, "ipc-channel", (candidate) => candidate.properties.side === side);
+    if (observation === undefined)
+        return undefined;
+    const { channel, operation } = observation.properties;
+    const source = observationSource(observation);
+    if (channel === undefined ||
+        operation === undefined ||
+        source === undefined)
+        return undefined;
+    return { channel, operation, source };
+};
+const preload = (value) => {
+    const observation = singleObservation(value, "electron-preload", (candidate) => candidate.properties.mechanism === "BrowserWindow:webPreferences.preload");
+    const path = observation?.properties.resolved_path;
+    const resolutionStatus = observation?.properties.resolution_status;
+    if (path === undefined || resolutionStatus === undefined)
+        return undefined;
+    return { path, resolution_status: resolutionStatus };
+};
+const headingChange = (value) => {
+    const result = javaScriptExportShapeComparisonResultSchema.safeParse(value);
+    if (!result.success)
+        return undefined;
+    const changes = result.data.changes.filter((change) => change.path === "/depth");
+    const change = changes.length === 1 ? changes[0] : undefined;
+    if (change === undefined)
+        return undefined;
+    return {
+        status: change.status,
+        path: change.path,
+        discriminant: change.discriminant,
+        left: change.left,
+        right: change.right,
+    };
+};
+const comparisonCoverage = (value) => {
+    const result = javaScriptExportShapeComparisonResultSchema.safeParse(value);
+    if (!result.success)
+        return undefined;
+    return {
+        status: result.data.coverage.status,
+        paired_variants: result.data.coverage.paired_variants,
+        ...result.data.summary,
+    };
+};
+const runtimeSemantics = (value) => {
+    const result = javaScriptExportShapeComparisonResultSchema.safeParse(value);
+    return result.success &&
+        result.data.limitations.includes("Return shapes are inferred from inert syntax and do not prove runtime behavior.")
+        ? "not_established"
+        : undefined;
+};
+const asarClaims = (targets) => {
+    const source = {
+        operation: "analyze_javascript_application",
+        authority: "shipped-artifact",
+        confidence: "derived",
+        subject: { path: targets.javascript, sha256: targets.javascriptSha256 },
+    };
+    return [
+        {
+            id: "renderer_api",
+            expectedValue: {
+                key: "profileApi",
+                members: ["read"],
+                world: "main",
+                source: "preload.js",
+            },
+            source: { ...source, select: rendererApi },
+        },
+        {
+            id: "renderer_ipc",
+            expectedValue: {
+                channel: "profile:read",
+                operation: "invoke",
+                source: "preload.js",
+            },
+            source: { ...source, select: ipcOperation("renderer") },
+        },
+        {
+            id: "main_ipc",
+            expectedValue: {
+                channel: "profile:read",
+                operation: "handle",
+                source: "main.js",
+            },
+            source: { ...source, select: ipcOperation("main") },
+        },
+        {
+            id: "preload",
+            expectedValue: { path: "preload.js", resolution_status: "resolved" },
+            source: { ...source, select: preload },
+        },
+    ];
+};
+const parserClaims = (targets) => {
+    const source = {
+        operation: "compare_javascript_export_shapes",
+        authority: "analyst-inference",
+        confidence: "inferred",
+        parameters: {
+            left_module_path: "parser.mjs",
+            left_export_name: "default",
+            right_module_path: "parser.mjs",
+            right_export_name: "default",
+        },
+        linkedSubjects: [
+            {
+                operation: "analyze_javascript_application",
+                path: targets.javascriptShapeLeft,
+            },
+            {
+                operation: "analyze_javascript_application",
+                path: targets.javascriptShapeRight,
+            },
+        ],
+    };
+    return [
+        {
+            id: "heading_change",
+            expectedValue: {
+                status: "added",
+                path: "/depth",
+                discriminant: { path: "/type", value: "heading" },
+                left: { availability: "absent" },
+                right: { availability: "literal", value: 1 },
+            },
+            source: { ...source, select: headingChange },
+        },
+        {
+            id: "comparison_coverage",
+            expectedValue: {
+                status: "complete-within-inputs",
+                paired_variants: 3,
+                added: 1,
+                removed: 0,
+                changed: 0,
+                unknown: 0,
+            },
+            source: { ...source, select: comparisonCoverage },
+        },
+        {
+            id: "runtime_semantics",
+            expectedValue: "not_established",
+            source: { ...source, select: runtimeSemantics },
+        },
+    ];
+};
+/** Return the closed known-answer rubric, or undefined for routing-only scenarios. */
+export const agentFixtureClaims = (scenarioId, targets) => {
+    if (scenarioId === "asar")
+        return asarClaims(targets);
+    if (scenarioId === "javascript-export-shape")
+        return parserClaims(targets);
+    return undefined;
+};
+const answerValueDescriptions = {
+    asar: [
+        'renderer_api: {"key": string, "members": string[], "world": string, "source": string}, describing the exposed bridge API, its members, world, and source file.',
+        'renderer_ipc: {"channel": string, "operation": string, "source": string}, describing the renderer-side IPC operation and its source file.',
+        'main_ipc: {"channel": string, "operation": string, "source": string}, describing the main-side IPC operation and its source file.',
+        'preload: {"path": string, "resolution_status": string}, describing the BrowserWindow-declared preload resolved within the shipped artifact.',
+    ],
+    "javascript-export-shape": [
+        'heading_change: {"status": string, "path": string, "discriminant": object or null, "left": object, "right": object}, copying the exact heading-change fields from the comparison result.',
+        'comparison_coverage: {"status": string, "paired_variants": number, "added": number, "removed": number, "changed": number, "unknown": number}, copying comparison coverage and summary counts.',
+        'runtime_semantics: "established" or "not_established", stating whether the cited comparison establishes runtime behavior.',
+    ],
+};
+/** Describe the strict answer contract without revealing any golden fixture values. */
+export const agentFixtureAnswerInstructions = (scenarioId) => {
+    const descriptions = answerValueDescriptions[scenarioId];
+    if (descriptions === undefined)
+        return "";
+    return [
+        "For this benchmark, return only one JSON object, without Markdown fences or surrounding prose.",
+        'The exact shape is {"claims": [{"id": string, "value": JSON value, "evidence_id": string, "confidence": string, "authority": string}]}.',
+        "Include exactly one claim per ID below, without additional fields or claims. Infer values only from REA tool results; do not read target files directly.",
+        "Each evidence_id must identify the successful REA result that supports its claim. Copy confidence and authority from that result's Evidence envelope.",
+        ...descriptions,
+    ].join("\n");
+};

package/dist/evaluation/AgentEvaluationReport.js

@@ -0,0 +1,32 @@
+/** Require configured facts to pass while preserving workflow and execution gates. */
+export const agentEvaluationPassed = (result) => result.exitCode === 0 &&
+    (result.expectedFirstTool === null
+        ? !result.naturalUse
+        : result.naturalUse) &&
+    result.correctFirstTool &&
+    result.repeatedCallCount === 0 &&
+    result.inputValidationFailureCount === 0 &&
+    result.requiredToolSubsequenceMet &&
+    result.inputTokens > 0 &&
+    (result.expectedFirstTool === null
+        ? result.targetClarificationPassed === true
+        : result.factualCorrectness === "passed" ||
+            (result.factualCorrectness === "not_assessed" &&
+                result.answerHeuristicsMet &&
+                result.epistemicCuePresent));
+/** Keep assessed fixture outcomes and scenarios without truth oracles distinct. */
+export const summarizeFactualCorrectness = (results) => {
+    const passed = results.filter(({ factualCorrectness }) => factualCorrectness === "passed").length;
+    const failed = results.filter(({ factualCorrectness }) => factualCorrectness === "failed").length;
+    const status = failed > 0 ? "failed" : passed > 0 ? "passed" : "not_assessed";
+    return {
+        status,
+        scope: "configured_fixture_claims",
+        counts: {
+            assessed: passed + failed,
+            passed,
+            failed,
+            notAssessed: results.length - passed - failed,
+        },
+    };
+};

package/dist/evaluation/CodexAgentEval.js

@@ -0,0 +1,188 @@
+import { deliveredCodexEvidence, evaluateKnownAnswers, } from "./KnownAnswerEvaluation.js";
+import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
+import { TOOL_CONTRACTS } from "../contracts/toolContracts.js";
+const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+const record = (value) => isRecord(value) ? value : undefined;
+const textValue = (value) => typeof value === "string" && value.length > 0 ? value : undefined;
+const numberValue = (value) => typeof value === "number" && Number.isFinite(value) ? value : 0;
+const canonicalValue = (value) => {
+    if (Array.isArray(value))
+        return value.map(canonicalValue);
+    const object = record(value);
+    if (object === undefined)
+        return value;
+    return Object.fromEntries(Object.entries(object)
+        .sort(([left], [right]) => compareUnicodeCodePoints(left, right))
+        .map(([key, child]) => [key, canonicalValue(child)]));
+};
+const toolResultError = (item) => {
+    const result = record(item.result ?? item.output);
+    const structured = record(result?.structured_content ?? result?.structuredContent);
+    const direct = record(structured?.error ?? result?.error ?? item.error);
+    if (direct !== undefined || structured !== undefined)
+        return direct;
+    // Codex preserves some REA failures only as one complete JSON text block.
+    // A client-truncated preview or SDK prose is not a structured error record.
+    const content = result?.content;
+    if (!Array.isArray(content) || content.length !== 1)
+        return undefined;
+    const block = record(content[0]);
+    if (block?.type !== "text" || typeof block.text !== "string")
+        return undefined;
+    try {
+        return record(record(JSON.parse(block.text))?.error);
+    }
+    catch (cause) {
+        if (cause instanceof SyntaxError || cause instanceof RangeError)
+            return undefined;
+        throw cause;
+    }
+};
+const toolResultFailed = (item, error) => {
+    if (item.status === "failed" ||
+        (item.error !== undefined && item.error !== null))
+        return true;
+    const result = record(item.result ?? item.output);
+    const structured = record(result?.structured_content ?? result?.structuredContent);
+    return (result?.isError === true ||
+        structured?.error !== undefined ||
+        error !== undefined);
+};
+const callFromEvent = (event) => {
+    const item = record(event.type === "item.completed"
+        ? event.item
+        : event.type === "mcp_tool_call"
+            ? event
+            : undefined);
+    if (item?.type !== "mcp_tool_call")
+        return undefined;
+    const tool = textValue(item.tool) ?? textValue(item.name);
+    if (tool === undefined)
+        return undefined;
+    const error = toolResultError(item);
+    return {
+        id: textValue(item.id) ?? null,
+        server: textValue(item.server) ?? textValue(item.server_name) ?? null,
+        tool,
+        arguments: item.arguments ?? item.input ?? {},
+        evidenceIds: deliveredCodexEvidence(event).map(({ evidence_id }) => evidence_id),
+        error: toolResultFailed(item, error),
+        errorCode: textValue(error?.code) ?? null,
+    };
+};
+const agentTextFromItem = (value) => {
+    const item = record(value);
+    if (item?.type !== "agent_message")
+        return undefined;
+    const direct = textValue(item.text) ?? textValue(item.message);
+    if (direct !== undefined)
+        return direct;
+    if (!Array.isArray(item.content))
+        return undefined;
+    const parts = item.content.flatMap((part) => {
+        const object = record(part);
+        const text = object === undefined ? undefined : textValue(object.text);
+        return text === undefined ? [] : [text];
+    });
+    return parts.length === 0 ? undefined : parts.join("\n");
+};
+const consumeEvent = (state, value) => {
+    const event = record(value);
+    if (event === undefined)
+        return;
+    const completedItem = event.type === "item.completed" ? event.item : undefined;
+    const call = callFromEvent(event);
+    if (call !== undefined) {
+        const identity = call.id ??
+            `${call.server ?? ""}:${call.tool}:${JSON.stringify(canonicalValue(call.arguments))}`;
+        if (!state.seenCallIds.has(identity)) {
+            state.seenCallIds.add(identity);
+            state.calls.push(call);
+        }
+    }
+    const message = agentTextFromItem(completedItem);
+    if (message !== undefined)
+        state.finalMessages.push(message);
+    if (event.type !== "turn.completed")
+        return;
+    const usage = record(event.usage);
+    state.inputTokens = Math.max(state.inputTokens, numberValue(usage?.input_tokens));
+    state.cachedInputTokens = Math.max(state.cachedInputTokens, numberValue(usage?.cached_input_tokens));
+    state.outputTokens = Math.max(state.outputTokens, numberValue(usage?.output_tokens));
+};
+/** Measure agent routing, repetition, model usage, and explicitly limited answer heuristics. */
+export const evaluateCodexEvents = (events, expectedFirstTool, options = {}) => {
+    const state = {
+        calls: [],
+        seenCallIds: new Set(),
+        finalMessages: [],
+        inputTokens: 0,
+        cachedInputTokens: 0,
+        outputTokens: 0,
+    };
+    for (const event of events)
+        consumeEvent(state, event);
+    const reaCalls = state.calls.filter(({ server }) => server === null || server.toLowerCase() === "rea");
+    const signatures = new Map();
+    let targetRevision = 0;
+    for (const call of reaCalls) {
+        // A status read before and after a successful target lifecycle change is
+        // useful verification, not a duplicate analysis or unchanged failed retry.
+        const context = call.tool === "binary_session" ? `${String(targetRevision)}:` : "";
+        const signature = `${context}${call.tool}:${JSON.stringify(canonicalValue(call.arguments))}`;
+        signatures.set(signature, (signatures.get(signature) ?? 0) + 1);
+        if (!call.error &&
+            (call.tool === "open_binary" || call.tool === "close_binary"))
+            targetRevision += 1;
+    }
+    const repeatedCallCount = [...signatures.values()].reduce((total, count) => total + Math.max(0, count - 1), 0);
+    const inputValidationFailureCount = reaCalls.filter((call) => call.errorCode === "invalid_request" ||
+        (call.error &&
+            TOOL_CONTRACTS.find(({ name }) => name === call.tool)?.inputSchema.safeParse(call.arguments).success === false)).length;
+    const requiredToolSubsequenceMet = containsOrderedSubsequence(reaCalls.filter(({ error }) => !error).map(({ tool }) => tool), options.requiredToolSubsequence ?? []);
+    const finalMessage = state.finalMessages.at(-1) ?? "";
+    const evidenceIds = [
+        ...new Set(reaCalls.flatMap(({ evidenceIds: ids }) => ids)),
+    ];
+    const finalCitesEvidence = evidenceIds.some((evidenceId) => finalMessage.includes(evidenceId));
+    const normalizedFinalMessage = finalMessage.toLocaleLowerCase("en-US");
+    const answerTermCoverageMet = (options.requiredAnswerTermGroups ?? []).every((terms) => terms.some((term) => normalizedFinalMessage.includes(term.toLocaleLowerCase("en-US"))));
+    const epistemicCuePresent = /\b(evidence|observed|inferred|unknown|unavailable|limitation|authority|not configured|could not|requires approval)\b/iu.test(finalMessage);
+    const factualAssessment = evaluateKnownAnswers(events, finalMessage, options.fixtureClaims);
+    return {
+        naturalUse: reaCalls.length > 0,
+        correctFirstTool: (reaCalls[0]?.tool ?? null) === expectedFirstTool,
+        firstTool: reaCalls[0]?.tool ?? null,
+        reaCalls,
+        repeatedCallCount,
+        inputValidationFailureCount,
+        requiredToolSubsequenceMet,
+        inputTokens: state.inputTokens,
+        cachedInputTokens: state.cachedInputTokens,
+        outputTokens: state.outputTokens,
+        finalMessage,
+        evidenceIds,
+        finalCitesEvidence,
+        answerTermCoverageMet,
+        answerHeuristicsMet: finalMessage.trim().length >= 80 &&
+            epistemicCuePresent &&
+            answerTermCoverageMet &&
+            requiredToolSubsequenceMet &&
+            (options.forbidInputValidationFailures !== true ||
+                inputValidationFailureCount === 0) &&
+            (options.requireEvidence !== true || finalCitesEvidence),
+        epistemicCuePresent,
+        factualCorrectness: factualAssessment.status,
+        factualAssessment,
+    };
+};
+const containsOrderedSubsequence = (values, required) => {
+    let index = 0;
+    for (const value of values) {
+        if (value === required[index])
+            index += 1;
+        if (index === required.length)
+            return true;
+    }
+    return required.length === 0;
+};