veryfront 0.1.874 → 0.1.876

package/esm/src/agent/testing/live-evals/runner.d.ts

@@ -0,0 +1,135 @@
+import { type AgUiSseProgressSnapshot as EvalProgressSnapshot, type ParsedAgUiSseRun as ParsedRun } from "../../index.js";
+import { type LiveEvalRuntime } from "./performance.js";
+import { type LiveEvalCaseMetadata } from "./report.js";
+import { type LiveEvalResultRecord } from "./result.js";
+/** Input payload for prepared live eval. */
+export interface PreparedLiveEvalInput {
+    prompt?: string;
+    metadata?: Record<string, string>;
+    verificationContext?: LiveEvalContext;
+    cleanup?: () => Promise<void>;
+    startSidecar?: () => Promise<(() => Promise<void>) | void>;
+}
+/** Context for live eval. */
+export interface LiveEvalContext {
+    apiUrl: string;
+    authToken: string;
+    projectId: string | null;
+}
+/** Public API contract for live eval case. */
+export interface LiveEvalCase {
+    readonly id: string;
+    readonly label: string;
+    readonly prompt?: string;
+    allowedTools?: string[];
+    forceRuntimeOverrides?: boolean;
+    requireProject?: boolean;
+    maxSteps?: number;
+    expectedEventSubsequence?: string[];
+    metadata?: LiveEvalCaseMetadata;
+    prepare?: (context: LiveEvalContext) => Promise<PreparedLiveEvalInput>;
+    verify: (run: ParsedRun, prepared: PreparedLiveEvalInput | null) => string | null | Promise<string | null>;
+}
+interface FileCheckInput {
+    filePath: string;
+    requiredContent?: string[];
+    description?: string;
+}
+/** Public API contract for live eval project file. */
+export interface LiveEvalProjectFile {
+    path: string;
+    content: string;
+}
+/** Input payload for live eval project file reader. */
+export interface LiveEvalProjectFileReaderInput {
+    filePath: string;
+    requestTimeoutMs: number;
+}
+/** Configuration used by live eval runner. */
+export interface LiveEvalRunnerConfig {
+    endpoint: string;
+    authToken: string;
+    apiUrl: string;
+    projectId: string | null;
+    branchId: string | null;
+    model: string | null;
+    requestTimeoutMs: number;
+    progressLogIntervalMs: number;
+    enableLlmJudge: boolean;
+    fetch?: (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+    log?: (message: string) => void;
+    readProjectFile?: (input: LiveEvalProjectFileReaderInput) => Promise<LiveEvalProjectFile | null>;
+}
+interface LiveEvalJudgeInput {
+    question: string;
+    criteria: string;
+}
+interface LiveEvalJudgeRequest extends LiveEvalJudgeInput {
+    answer: string;
+}
+interface LiveEvalJudgeResult {
+    pass: boolean;
+    reason: string;
+}
+declare function collectPreparedArtifactPaths(prepared: PreparedLiveEvalInput | null): string[];
+interface LiveEvalResultContext {
+    id: string;
+    label: string;
+    runtime: LiveEvalRuntime;
+    startedAt: number;
+    conversationId?: string | null;
+    artifactPaths?: string[];
+}
+interface LiveEvalRunArtifactsInput {
+    run: ParsedRun;
+    runId?: string;
+    traceSignature: string;
+}
+interface LiveEvalRunArtifacts {
+    runId?: string;
+    traceSignature: string;
+    toolStarts: string[];
+    toolArgsPreview: string;
+    textPreview: string;
+}
+declare function createLiveEvalRunArtifacts(input: LiveEvalRunArtifactsInput): LiveEvalRunArtifacts;
+declare function createFailedRunEvalResult(input: {
+    details: string;
+    context: LiveEvalResultContext;
+    runArtifacts: LiveEvalRunArtifacts;
+}): LiveEvalResultRecord;
+declare function createPassedRunEvalResult(input: {
+    details: string;
+    context: LiveEvalResultContext;
+    runArtifacts: LiveEvalRunArtifacts;
+}): LiveEvalResultRecord;
+declare function createStreamingFailureEvalResult(input: {
+    details: string;
+    context: LiveEvalResultContext;
+    progress: EvalProgressSnapshot;
+}): LiveEvalResultRecord;
+declare function extractRunId(run: ParsedRun): string | null;
+/** Check whether finished is present. */
+export declare function hasFinished(run: ParsedRun): boolean;
+/** Contains skill load helper. */
+export declare function containsSkillLoad(run: ParsedRun, skillId: string): boolean;
+/** Count step started events helper. */
+export declare function countStepStartedEvents(run: ParsedRun): number;
+/** Create live eval case support. */
+export declare function createLiveEvalCaseSupport(config: LiveEvalRunnerConfig): {
+    runEval: (testCase: LiveEvalCase, runtime: LiveEvalRuntime) => Promise<LiveEvalResultRecord>;
+    verifyFileExists: (input: FileCheckInput) => Promise<string | null>;
+    withJudge: (structuralVerify: (run: ParsedRun) => string | null, judgeInput: LiveEvalJudgeInput) => (run: ParsedRun) => Promise<string | null>;
+    judgeLlm: (input: LiveEvalJudgeRequest) => Promise<LiveEvalJudgeResult>;
+};
+/** White-box helpers used by live eval runner tests. */
+export declare const liveEvalRunnerInternals: {
+    collectPreparedArtifactPaths: typeof collectPreparedArtifactPaths;
+    createFailedRunEvalResult: typeof createFailedRunEvalResult;
+    createLiveEvalRunArtifacts: typeof createLiveEvalRunArtifacts;
+    createPassedRunEvalResult: typeof createPassedRunEvalResult;
+    createStreamingFailureEvalResult: typeof createStreamingFailureEvalResult;
+    extractRunId: typeof extractRunId;
+};
+export {};
+//# sourceMappingURL=runner.d.ts.map

package/esm/src/agent/testing/live-evals/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/runner.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,uBAAuB,IAAI,oBAAoB,EAIpD,KAAK,gBAAgB,IAAI,SAAS,EACnC,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAExD,OAAO,EAAE,KAAK,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAIL,KAAK,oBAAoB,EAC1B,MAAM,aAAa,CAAC;AAErB,4CAA4C;AAC5C,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,mBAAmB,CAAC,EAAE,eAAe,CAAC;IACtC,OAAO,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;CAC5D;AAED,6BAA6B;AAC7B,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,8CAA8C;AAC9C,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wBAAwB,CAAC,EAAE,MAAM,EAAE,CAAC;IACpC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;IAChC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACvE,MAAM,EAAE,CACN,GAAG,EAAE,SAAS,EACd,QAAQ,EAAE,qBAAqB,GAAG,IAAI,KACnC,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAC7C;AAED,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,sDAAsD;AACtD,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,uDAAuD;AACvD,MAAM,WAAW,8BAA8B;IAC7C,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,8CAA8C;AAC9C,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,cAAc,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjF,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,8BAA8B,KAAK,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;CAClG;AAED,UAAU,kBAAkB;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,oBAAqB,SAAQ,kBAAkB;IACvD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,mBAAmB;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAyJD,iBAAS,4BAA4B,CAAC,QAAQ,EAAE,qBAAqB,GAAG,IAAI,GAAG,MAAM,EAAE,CAYtF;AASD,UAAU,qBAAqB;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,eAAe,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,UAAU,yBAAyB;IACjC,GAAG,EAAE,SAAS,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,UAAU,oBAAoB;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,iBAAS,0BAA0B,CAAC,KAAK,EAAE,yBAAyB,GAAG,oBAAoB,CAQ1F;AAED,iBAAS,yBAAyB,CAAC,KAAK,EAAE;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,YAAY,EAAE,oBAAoB,CAAC;CACpC,GAAG,oBAAoB,CAevB;AAED,iBAAS,yBAAyB,CAAC,KAAK,EAAE;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,YAAY,EAAE,oBAAoB,CAAC;CACpC,GAAG,oBAAoB,CAevB;AAED,iBAAS,gCAAgC,CAAC,KAAK,EAAE;IAC/C,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,QAAQ,EAAE,oBAAoB,CAAC;CAChC,GAAG,oBAAoB,CAcvB;AA+FD,iBAAS,YAAY,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,GAAG,IAAI,CASnD;AAED,yCAAyC;AACzC,wBAAgB,WAAW,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAEnD;AAED,kCAAkC;AAClC,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAE1E;AAED,wCAAwC;AACxC,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,CAE7D;AAED,qCAAqC;AACrC,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,oBAAoB,GAAG;IACvE,OAAO,EAAE,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC7F,gBAAgB,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACpE,SAAS,EAAE,CACT,gBAAgB,EAAE,CAAC,GAAG,EAAE,SAAS,KAAK,MAAM,GAAG,IAAI,EACnD,UAAU,EAAE,kBAAkB,KAC3B,CAAC,GAAG,EAAE,SAAS,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAChD,QAAQ,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,OAAO,CAAC,mBAAmB,CAAC,CAAC;CACzE,CAoIA;AAED,wDAAwD;AACxD,eAAO,MAAM,uBAAuB;;;;;;;CAOnC,CAAC"}

package/esm/src/agent/testing/live-evals/runner.js

@@ -0,0 +1,396 @@
+import * as dntShim from "../../../../_dnt.shims.js";
+import { agUiSseEventTypes, buildAgUiSseTraceSignature as buildTraceSignature, getAgUiSseStringField as getStringField, parseAgUiSseResponse as parseSseResponse, } from "../../index.js";
+import { buildFailureSuffix, buildProgressLine, containsOrderedSubsequence } from "./formatting.js";
+import { buildLiveEvalRequestBody } from "./request.js";
+import { createFailedEvalResult, createPassedEvalResult, createSkippedEvalResult, } from "./result.js";
+function resolveFetch(config) {
+    return config.fetch ?? fetch;
+}
+function createLiveEvalJudgeSupport(config) {
+    async function judgeLlm(input) {
+        try {
+            const body = buildLiveEvalRequestBody({
+                testCaseId: "llm-judge",
+                prompt: `You are an eval judge. Grade the following answer.
+
+QUESTION: ${input.question}
+
+ANSWER: ${input.answer}
+
+CRITERIA: ${input.criteria}
+
+Respond with exactly one line: PASS or FAIL followed by a brief reason.
+Example: "PASS — correctly explains the pattern with accurate details"
+Example: "FAIL — mentions the wrong file convention"`,
+                projectId: null,
+                allowedTools: [],
+                forceRuntimeOverrides: true,
+                maxSteps: 2,
+            });
+            const response = await resolveFetch(config)(config.endpoint, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${config.authToken}`,
+                },
+                body: JSON.stringify(body),
+                signal: AbortSignal.timeout(30_000),
+            });
+            const run = await parseSseResponse(response);
+            if (run.responseStatus !== 200) {
+                return { pass: false, reason: `judge returned HTTP ${run.responseStatus}` };
+            }
+            const line = run.text
+                .split("\n")
+                .map((value) => value.trim())
+                .find((value) => value.length > 0) ?? "";
+            if (line.toUpperCase().startsWith("PASS")) {
+                return { pass: true, reason: line };
+            }
+            return { pass: false, reason: line || "judge returned no decision" };
+        }
+        catch (error) {
+            return {
+                pass: false,
+                reason: error instanceof Error ? error.message : String(error),
+            };
+        }
+    }
+    function withJudge(structuralVerify, judgeInput) {
+        return async (run) => {
+            const structuralFailure = structuralVerify(run);
+            if (structuralFailure) {
+                return structuralFailure;
+            }
+            if (!config.enableLlmJudge) {
+                return null;
+            }
+            const judgment = await judgeLlm({
+                question: judgeInput.question,
+                answer: run.text,
+                criteria: judgeInput.criteria,
+            });
+            return judgment.pass ? null : `LLM judge: ${judgment.reason}`;
+        };
+    }
+    return {
+        judgeLlm,
+        withJudge,
+    };
+}
+function createInitialProgressSnapshot() {
+    return {
+        eventCount: 0,
+        lastEventType: null,
+        lastToolCallName: null,
+        toolStarts: [],
+        textLength: 0,
+    };
+}
+function isUnrefableTimer(value) {
+    return typeof value === "object" && value !== null && "unref" in value &&
+        typeof value.unref === "function";
+}
+function maybeUnrefTimer(timer) {
+    if (isUnrefableTimer(timer)) {
+        timer.unref();
+    }
+}
+function createLiveEvalProgressReporter(input) {
+    let latestProgress = createInitialProgressSnapshot();
+    const progressTimer = dntShim.setInterval(() => {
+        input.log(buildProgressLine({
+            caseId: input.caseId,
+            startedAt: input.startedAt,
+            progress: latestProgress,
+        }));
+    }, input.intervalMs);
+    maybeUnrefTimer(progressTimer);
+    return {
+        stop: () => {
+            clearInterval(progressTimer);
+        },
+        update: (snapshot) => {
+            latestProgress = snapshot;
+        },
+        getSnapshot: () => latestProgress,
+    };
+}
+function collectPreparedArtifactPaths(prepared) {
+    if (!prepared?.metadata) {
+        return [];
+    }
+    return [
+        ...new Set(Object.entries(prepared.metadata)
+            .filter(([key, value]) => key.toLowerCase().includes("path") && value.length > 0)
+            .map(([, value]) => value)),
+    ].sort();
+}
+function extractPreparedConversationId(prepared) {
+    return typeof prepared?.metadata?.conversationId === "string" &&
+        prepared.metadata.conversationId.length > 0
+        ? prepared.metadata.conversationId
+        : null;
+}
+function createLiveEvalRunArtifacts(input) {
+    return {
+        ...(input.runId ? { runId: input.runId } : {}),
+        traceSignature: input.traceSignature,
+        toolStarts: input.run.toolStarts,
+        toolArgsPreview: input.run.toolArgs.join(" | ").slice(0, 1000),
+        textPreview: input.run.text.slice(0, 280),
+    };
+}
+function createFailedRunEvalResult(input) {
+    return createFailedEvalResult({
+        id: input.context.id,
+        label: input.context.label,
+        runtime: input.context.runtime,
+        details: input.details,
+        startedAt: input.context.startedAt,
+        ...(input.context.conversationId ? { conversationId: input.context.conversationId } : {}),
+        ...(input.runArtifacts.runId ? { runId: input.runArtifacts.runId } : {}),
+        ...(input.context.artifactPaths?.length ? { artifactPaths: input.context.artifactPaths } : {}),
+        traceSignature: input.runArtifacts.traceSignature,
+        toolStarts: input.runArtifacts.toolStarts,
+        toolArgsPreview: input.runArtifacts.toolArgsPreview,
+        textPreview: input.runArtifacts.textPreview,
+    });
+}
+function createPassedRunEvalResult(input) {
+    return createPassedEvalResult({
+        id: input.context.id,
+        label: input.context.label,
+        runtime: input.context.runtime,
+        details: input.details,
+        startedAt: input.context.startedAt,
+        ...(input.context.conversationId ? { conversationId: input.context.conversationId } : {}),
+        ...(input.runArtifacts.runId ? { runId: input.runArtifacts.runId } : {}),
+        ...(input.context.artifactPaths?.length ? { artifactPaths: input.context.artifactPaths } : {}),
+        traceSignature: input.runArtifacts.traceSignature,
+        toolStarts: input.runArtifacts.toolStarts,
+        toolArgsPreview: input.runArtifacts.toolArgsPreview,
+        textPreview: input.runArtifacts.textPreview,
+    });
+}
+function createStreamingFailureEvalResult(input) {
+    return createFailedEvalResult({
+        id: input.context.id,
+        label: input.context.label,
+        runtime: input.context.runtime,
+        details: `${input.details}${buildFailureSuffix(input.progress)}`,
+        startedAt: input.context.startedAt,
+        ...(input.context.conversationId ? { conversationId: input.context.conversationId } : {}),
+        ...(input.context.artifactPaths?.length ? { artifactPaths: input.context.artifactPaths } : {}),
+        toolStarts: input.progress.toolStarts,
+        textPreview: input.progress.textLength > 0
+            ? `${input.progress.textLength} characters streamed`
+            : undefined,
+    });
+}
+function createLiveEvalResultContext(input) {
+    return {
+        id: input.testCase.id,
+        label: input.testCase.label,
+        runtime: input.runtime,
+        startedAt: input.startedAt,
+        ...(input.conversationId ? { conversationId: input.conversationId } : {}),
+        ...(input.artifactPaths.length > 0 ? { artifactPaths: input.artifactPaths } : {}),
+    };
+}
+function buildLiveEvalRunBody(input) {
+    const customBody = typeof input.prepared?.metadata?.customBody === "string"
+        ? input.prepared.metadata.customBody
+        : null;
+    if (customBody) {
+        return JSON.parse(customBody);
+    }
+    return buildLiveEvalRequestBody({
+        testCaseId: input.testCase.id,
+        prompt: input.prepared?.prompt ?? input.testCase.prompt ?? "",
+        metadata: input.prepared?.metadata,
+        projectId: input.config.projectId && input.testCase.requireProject
+            ? input.config.projectId
+            : null,
+        ...(input.config.branchId ? { branchId: input.config.branchId } : {}),
+        ...(input.config.model ? { model: input.config.model } : {}),
+        ...(input.conversationId ? { conversationId: input.conversationId } : {}),
+        allowedTools: input.testCase.allowedTools,
+        forceRuntimeOverrides: input.testCase.forceRuntimeOverrides,
+        maxSteps: input.testCase.maxSteps,
+    });
+}
+async function resolveCompletedLiveEvalRun(input) {
+    const traceSignature = buildTraceSignature(input.run.eventTypes);
+    const runArtifacts = createLiveEvalRunArtifacts({
+        run: input.run,
+        runId: input.runId,
+        traceSignature,
+    });
+    const failure = await input.testCase.verify(input.run, input.prepared);
+    if (!failure && input.testCase.expectedEventSubsequence) {
+        if (!containsOrderedSubsequence(input.run.eventTypes, input.testCase.expectedEventSubsequence)) {
+            return createFailedRunEvalResult({
+                context: input.context,
+                details: `Expected AG-UI event subsequence ${input.testCase.expectedEventSubsequence.join(" -> ")}, got ${traceSignature}`,
+                runArtifacts,
+            });
+        }
+    }
+    if (failure) {
+        return createFailedRunEvalResult({
+            context: input.context,
+            details: failure,
+            runArtifacts,
+        });
+    }
+    return createPassedRunEvalResult({
+        context: input.context,
+        details: `OK: ${input.run.toolStarts.join(", ") || "no tools"} | ${input.run.text.slice(0, 140) || "no text"}`,
+        runArtifacts,
+    });
+}
+function extractRunId(run) {
+    for (const event of run.events) {
+        const runId = getStringField(event, "runId") ?? getStringField(event, "run_id");
+        if (runId) {
+            return runId;
+        }
+    }
+    return null;
+}
+/** Check whether finished is present. */
+export function hasFinished(run) {
+    return run.eventTypes.includes(agUiSseEventTypes.runFinished) && !run.runError;
+}
+/** Contains skill load helper. */
+export function containsSkillLoad(run, skillId) {
+    return run.toolStarts.includes("load_skill") && run.toolArgs.join("").includes(skillId);
+}
+/** Count step started events helper. */
+export function countStepStartedEvents(run) {
+    return run.eventTypes.filter((eventType) => eventType === agUiSseEventTypes.stepStarted).length;
+}
+/** Create live eval case support. */
+export function createLiveEvalCaseSupport(config) {
+    const fetchImpl = resolveFetch(config);
+    const log = config.log ?? console.log;
+    const { judgeLlm, withJudge } = createLiveEvalJudgeSupport(config);
+    async function verifyFileExists(input) {
+        if (!config.projectId || !config.readProjectFile) {
+            return null;
+        }
+        const file = await config.readProjectFile({
+            filePath: input.filePath,
+            requestTimeoutMs: config.requestTimeoutMs,
+        });
+        if (!file) {
+            return `${input.description ?? input.filePath}: file not found in project after task completed`;
+        }
+        if (!file.content || file.content.trim().length === 0) {
+            return `${input.description ?? input.filePath}: file exists but is empty`;
+        }
+        if (input.requiredContent) {
+            const missing = input.requiredContent.filter((keyword) => !file.content.toLowerCase().includes(keyword.toLowerCase()));
+            if (missing.length > 0) {
+                return `${input.description ?? input.filePath}: missing required content: ${missing.join(", ")}. Got: ${file.content.slice(0, 200)}`;
+            }
+        }
+        return null;
+    }
+    async function runEval(testCase, runtime) {
+        const startedAt = Date.now();
+        if (testCase.requireProject && !config.projectId) {
+            return createSkippedEvalResult({
+                id: testCase.id,
+                label: testCase.label,
+                runtime,
+                details: "Skipped because AG_UI_EVAL_PROJECT_ID is not set.",
+                startedAt,
+            });
+        }
+        const prepared = testCase.prepare
+            ? await testCase.prepare({
+                apiUrl: config.apiUrl,
+                authToken: config.authToken,
+                projectId: config.projectId,
+            })
+            : null;
+        const preparedConversationId = extractPreparedConversationId(prepared);
+        const preparedArtifactPaths = collectPreparedArtifactPaths(prepared);
+        const resultContext = createLiveEvalResultContext({
+            testCase,
+            runtime,
+            startedAt,
+            conversationId: preparedConversationId,
+            artifactPaths: preparedArtifactPaths,
+        });
+        try {
+            const sidecarCleanup = prepared?.startSidecar ? await prepared.startSidecar() : undefined;
+            const progressReporter = createLiveEvalProgressReporter({
+                caseId: testCase.id,
+                startedAt,
+                intervalMs: config.progressLogIntervalMs,
+                log,
+            });
+            const body = buildLiveEvalRunBody({
+                config,
+                testCase,
+                prepared,
+                conversationId: preparedConversationId,
+            });
+            try {
+                const response = await fetchImpl(config.endpoint, {
+                    method: "POST",
+                    headers: {
+                        "Content-Type": "application/json",
+                        Authorization: `Bearer ${config.authToken}`,
+                    },
+                    body: JSON.stringify(body),
+                    signal: AbortSignal.timeout(config.requestTimeoutMs),
+                });
+                log(`[stream] ${runtime}:${testCase.id} HTTP ${response.status}`);
+                const run = await parseSseResponse(response, {
+                    onProgress: progressReporter.update,
+                });
+                return resolveCompletedLiveEvalRun({
+                    testCase,
+                    run,
+                    prepared,
+                    context: resultContext,
+                    runId: extractRunId(run) ?? undefined,
+                });
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                return createStreamingFailureEvalResult({
+                    context: resultContext,
+                    details: message,
+                    progress: progressReporter.getSnapshot(),
+                });
+            }
+            finally {
+                progressReporter.stop();
+                await sidecarCleanup?.();
+            }
+        }
+        finally {
+            await prepared?.cleanup?.();
+        }
+    }
+    return {
+        judgeLlm,
+        runEval,
+        verifyFileExists,
+        withJudge,
+    };
+}
+/** White-box helpers used by live eval runner tests. */
+export const liveEvalRunnerInternals = {
+    collectPreparedArtifactPaths,
+    createFailedRunEvalResult,
+    createLiveEvalRunArtifacts,
+    createPassedRunEvalResult,
+    createStreamingFailureEvalResult,
+    extractRunId,
+};

package/esm/src/chat/message-prep.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"message-prep.d.ts","sourceRoot":"","sources":["../../../src/src/chat/message-prep.ts"],"names":[],"mappings":"AAAA,OAAO,yBAAyB,CAAC;AASjC,OAAO,EAKL,KAAK,aAAa,EAGlB,KAAK,oBAAoB,EAE1B,MAAM,YAAY,CAAC;AAMpB,4EAA4E;AAC5E,MAAM,WAAW,iDAAiD;IAChE,sBAAsB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC5C;AAED,uBAAuB;AACvB,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAErD;AAOD,qBAAqB;AACrB,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,oBAAoB,EAAE,EAChC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,GACb,oBAAoB,EAAE,CAiCxB;AAqCD,kDAAkD;AAClD,wBAAgB,qCAAqC,CACnD,QAAQ,EAAE,oBAAoB,EAAE,EAChC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,GACf,oBAAoB,EAAE,CAiExB;AA+BD,4DAA4D;AAC5D,wBAAgB,6BAA6B,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAGxE;AAED,gDAAgD;AAChD,wBAAgB,kCAAkC,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,aAAa,EAAE,CAwB7F;AAED,qDAAqD;AACrD,wBAAgB,wCAAwC,CACtD,QAAQ,EAAE,aAAa,EAAE,GACxB,aAAa,EAAE,CAmDjB;AAyCD,gCAAgC;AAChC,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,aAAa,EAAE,CAiBhF;AAsFD,wCAAwC;AACxC,wBAAgB,6BAA6B,CAC3C,QAAQ,EAAE,oBAAoB,EAAE,GAC/B,oBAAoB,EAAE,CA8BxB;AA4ED,wDAAwD;AACxD,wBAAgB,0CAA0C,CACxD,QAAQ,EAAE,aAAa,EAAE,EACzB,OAAO,GAAE,iDAAsD,GAC9D,oBAAoB,EAAE,CAqBxB;AA2OD,6BAA6B;AAC7B,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,GAAG,oBAAoB,EAAE,CA4E3F;AAWD,yBAAyB;AACzB,wBAAgB,eAAe,CAAC,QAAQ,EAAE,oBAAoB,EAAE,GAAG,oBAAoB,EAAE,CA4IxF;AAED,yBAAyB;AACzB,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAGjF;AAED,sCAAsC;AACtC,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,GAAG,oBAAoB,EAAE,CA4B7F;AAED,wBAAwB;AACxB,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,oBAAoB,EAAE,EAChC,QAAQ,GAAE,MAAU,GACnB,oBAAoB,EAAE,CAexB;AAED,2BAA2B;AAC3B,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,GAAG,oBAAoB,EAAE,CAgE1F;AAED,4BAA4B;AAC5B,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,oBAAoB,EAAE,EAChC,MAAM,GAAE,MAA6B,EACrC,QAAQ,GAAE,MAAU,GACnB,oBAAoB,EAAE,CAMxB"}
+{"version":3,"file":"message-prep.d.ts","sourceRoot":"","sources":["../../../src/src/chat/message-prep.ts"],"names":[],"mappings":"AAAA,OAAO,yBAAyB,CAAC;AASjC,OAAO,EAKL,KAAK,aAAa,EAGlB,KAAK,oBAAoB,EAE1B,MAAM,YAAY,CAAC;AAMpB,4EAA4E;AAC5E,MAAM,WAAW,iDAAiD;IAChE,sBAAsB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC5C;AAED,uBAAuB;AACvB,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAErD;AAOD,qBAAqB;AACrB,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,oBAAoB,EAAE,EAChC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,GACb,oBAAoB,EAAE,CAiCxB;AAqCD,kDAAkD;AAClD,wBAAgB,qCAAqC,CACnD,QAAQ,EAAE,oBAAoB,EAAE,EAChC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,GACf,oBAAoB,EAAE,CAiExB;AA+BD,4DAA4D;AAC5D,wBAAgB,6BAA6B,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAGxE;AAED,gDAAgD;AAChD,wBAAgB,kCAAkC,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,aAAa,EAAE,CAwB7F;AAED,qDAAqD;AACrD,wBAAgB,wCAAwC,CACtD,QAAQ,EAAE,aAAa,EAAE,GACxB,aAAa,EAAE,CAmDjB;AAyCD,gCAAgC;AAChC,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,aAAa,EAAE,CAiBhF;AAsFD,wCAAwC;AACxC,wBAAgB,6BAA6B,CAC3C,QAAQ,EAAE,oBAAoB,EAAE,GAC/B,oBAAoB,EAAE,CA8BxB;AA4ED,wDAAwD;AACxD,wBAAgB,0CAA0C,CACxD,QAAQ,EAAE,aAAa,EAAE,EACzB,OAAO,GAAE,iDAAsD,GAC9D,oBAAoB,EAAE,CAqBxB;AAmRD,6BAA6B;AAC7B,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,GAAG,oBAAoB,EAAE,CA4E3F;AAWD,yBAAyB;AACzB,wBAAgB,eAAe,CAAC,QAAQ,EAAE,oBAAoB,EAAE,GAAG,oBAAoB,EAAE,CA4IxF;AAED,yBAAyB;AACzB,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAGjF;AAED,sCAAsC;AACtC,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,GAAG,oBAAoB,EAAE,CA4B7F;AAED,wBAAwB;AACxB,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,oBAAoB,EAAE,EAChC,QAAQ,GAAE,MAAU,GACnB,oBAAoB,EAAE,CAexB;AAED,2BAA2B;AAC3B,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,GAAG,oBAAoB,EAAE,CAgE1F;AAED,4BAA4B;AAC5B,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,oBAAoB,EAAE,EAChC,MAAM,GAAE,MAA6B,EACrC,QAAQ,GAAE,MAAU,GACnB,oBAAoB,EAAE,CAMxB"}

package/esm/src/chat/message-prep.js

@@ -532,13 +532,27 @@ function compactContactValue(value) {
         if (typeof emailAddress.address === "string")
             compact.address = emailAddress.address;
     }
-    for (const field of ["login", "name", "address", "email", "id"]) {
+    for (const field of ["login", "name", "displayName", "address", "email", "id", "accountId"]) {
         if (typeof value[field] === "string" || typeof value[field] === "number") {
             compact[field] = value[field];
         }
     }
     return Object.keys(compact).length > 0 ? compact : null;
 }
+function compactNamedArrayValue(value) {
+    if (typeof value === "string")
+        return value;
+    if (!isRecord(value))
+        return null;
+    for (const field of ["name", "displayName", "login", "key", "id"]) {
+        const candidate = value[field];
+        if (typeof candidate === "string" && candidate.length > 0)
+            return candidate;
+        if (typeof candidate === "number")
+            return String(candidate);
+    }
+    return null;
+}
 function compactHistoricalField(field, fieldValue) {
     if (field.kind === "contact") {
         return compactContactValue(fieldValue);
@@ -557,6 +571,14 @@ function compactHistoricalField(field, fieldValue) {
         const strings = fieldValue.filter((item) => typeof item === "string");
         return strings.length > 0 ? strings : null;
     }
+    if (field.kind === "named-array") {
+        if (!Array.isArray(fieldValue))
+            return null;
+        const names = fieldValue
+            .map((item) => compactNamedArrayValue(item))
+            .filter((item) => item !== null);
+        return names.length > 0 ? names : null;
+    }
     if (field.kind === "object") {
         if (!isRecord(fieldValue))
             return null;
@@ -580,12 +602,24 @@ function compactHistoricalField(field, fieldValue) {
     }
     return null;
 }
+function getHistoricalFieldValue(source, field) {
+    if (!field.path || field.path.length === 0) {
+        return source[field.name];
+    }
+    let value = source;
+    for (const segment of field.path) {
+        if (!isRecord(value))
+            return undefined;
+        value = value[segment];
+    }
+    return value;
+}
 function compactHistoricalItemValue(value, fields) {
     if (!isRecord(value))
         return null;
     const compact = {};
     for (const field of fields) {
-        const fieldValue = compactHistoricalField(field, value[field.name]);
+        const fieldValue = compactHistoricalField(field, getHistoricalFieldValue(value, field));
         if (fieldValue !== null)
             compact[field.name] = fieldValue;
     }
@@ -626,7 +660,7 @@ function compactHistoricalToolSummaryOutput(rawValue, contract) {
     };
     if (output && contract.outputFields) {
         for (const field of contract.outputFields) {
-            const fieldValue = compactHistoricalField(field, output[field.name]);
+            const fieldValue = compactHistoricalField(field, getHistoricalFieldValue(output, field));
             if (fieldValue !== null)
                 compacted[field.name] = fieldValue;
         }

package/esm/src/integrations/_data.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"_data.d.ts","sourceRoot":"","sources":["../../../src/src/integrations/_data.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAErD,eAAO,MAAM,UAAU,EAAE,iBAAiB,EAk61DzC,CAAC;AAEF,eAAO,MAAM,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAyZxC,CAAC"}
+{"version":3,"file":"_data.d.ts","sourceRoot":"","sources":["../../../src/src/integrations/_data.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAErD,eAAO,MAAM,UAAU,EAAE,iBAAiB,EAg/1DzC,CAAC;AAEF,eAAO,MAAM,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAyZxC,CAAC"}