veryfront 0.1.875 → 0.1.876

package/esm/src/agent/testing/live-evals/api-client.js

@@ -0,0 +1,313 @@
+import * as dntShim from "../../../../_dnt.shims.js";
+import { ensureBuiltinSchemaValidator } from "../../../extensions/builtin-extensions.js";
+import { defineSchema } from "../../../schemas/index.js";
+ensureBuiltinSchemaValidator();
+const getLiveEvalIdResponseSchema = defineSchema((v) => v.object({
+    id: v.string().optional(),
+}));
+const getProjectUploadResponseSchema = defineSchema((v) => v.object({
+    file_upload_url: v.string().optional(),
+    required_headers: v.record(v.string(), v.string()).optional(),
+}));
+const getProjectUploadListResponseSchema = defineSchema((v) => v.object({
+    data: v.array(v.object({ path: v.string().optional() }).passthrough()).optional(),
+}));
+const getProjectFileResponseSchema = defineSchema((v) => v.object({
+    path: v.string().optional(),
+    content: v.string().optional(),
+}));
+const getInputRequestRecordSchema = defineSchema((v) => v.object({
+    id: v.string(),
+    status: v.string(),
+}));
+const getInputRequestListResponseSchema = defineSchema((v) => v.object({
+    data: v.array(v.unknown()).optional(),
+}));
+function createLiveEvalAuthHeaders(context) {
+    const headers = new Headers();
+    headers.set("Authorization", `Bearer ${context.authToken}`);
+    return headers;
+}
+function createLiveEvalJsonHeaders(context) {
+    const headers = createLiveEvalAuthHeaders(context);
+    headers.set("Content-Type", "application/json");
+    return headers;
+}
+function requireLiveEvalProjectId(projectId, errorMessage) {
+    if (!projectId) {
+        throw new Error(errorMessage);
+    }
+    return projectId;
+}
+function createFetch(context) {
+    return context.fetch ?? fetch;
+}
+function createApiUrl(context, path) {
+    const baseHref = context.apiUrl.endsWith("/") ? context.apiUrl : `${context.apiUrl}/`;
+    const relativePath = path.startsWith("/") ? path.slice(1) : path;
+    return new URL(relativePath, baseHref);
+}
+function createProjectUploadHeaders(requiredHeaders, contentType) {
+    const uploadHeaders = new Headers(requiredHeaders);
+    if (!uploadHeaders.has("Content-Type")) {
+        uploadHeaders.set("Content-Type", contentType);
+    }
+    return uploadHeaders;
+}
+function getProjectUploadBodySize(body, explicitSize) {
+    if (typeof explicitSize === "number") {
+        return explicitSize;
+    }
+    if (typeof body === "string") {
+        return new TextEncoder().encode(body).byteLength;
+    }
+    if (body instanceof Blob) {
+        return body.size;
+    }
+    if (body instanceof URLSearchParams) {
+        return new TextEncoder().encode(body.toString()).byteLength;
+    }
+    if (body instanceof ArrayBuffer) {
+        return body.byteLength;
+    }
+    if (ArrayBuffer.isView(body)) {
+        return body.byteLength;
+    }
+    throw new Error("Project upload fixtures require size when body length cannot be inferred");
+}
+function createProjectUploadBody(body, contentType) {
+    if (body instanceof Blob) {
+        return body;
+    }
+    if (typeof body === "string") {
+        return new Blob([body], { type: contentType });
+    }
+    if (body instanceof Uint8Array) {
+        return new Blob([body.slice()], { type: contentType });
+    }
+    return body;
+}
+function getResponseText(response) {
+    return response.text();
+}
+async function wait(input) {
+    await new Promise((resolve) => {
+        dntShim.setTimeout(resolve, input.ms);
+    });
+}
+async function waitForProjectUploadFixture(context, input) {
+    const listUrl = createApiUrl(context, `/projects/${input.projectId}/uploads`);
+    const requestFetch = createFetch(context);
+    const maxAttempts = input.maxAttempts ?? 12;
+    const pollIntervalMs = input.pollIntervalMs ?? 2_000;
+    for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
+        const listResponse = await requestFetch(listUrl, {
+            headers: createLiveEvalAuthHeaders(context),
+            signal: AbortSignal.timeout(input.requestTimeoutMs),
+        });
+        if (!listResponse.ok) {
+            throw new Error(`Failed to confirm project upload fixture: ${listResponse.status} ${await getResponseText(listResponse)}`);
+        }
+        const payload = getProjectUploadListResponseSchema().parse(await listResponse.json());
+        if (payload.data?.some((upload) => upload.path === input.filePath)) {
+            return input.filePath;
+        }
+        if (attempt + 1 < maxAttempts) {
+            await wait({ ms: pollIntervalMs });
+        }
+    }
+    throw new Error(`Project upload fixture did not appear in time: ${input.filePath}`);
+}
+/** Create live eval API client. */
+export function createLiveEvalApiClient(context) {
+    return {
+        createConversation: (input) => createLiveEvalConversation(context, input),
+        deleteConversation: (input) => deleteLiveEvalConversation(context, input),
+        createProjectUploadFixture: (input) => createLiveEvalProjectUploadFixture(context, input),
+        getProjectFile: (input) => getLiveEvalProjectFile(context, input),
+        createRelease: (input) => createLiveEvalRelease(context, input),
+        deleteProjectFile: (input) => deleteLiveEvalProjectFile(context, input),
+        listOpenInputRequests: (input) => listOpenLiveEvalInputRequests(context, input),
+        waitForOpenInputRequest: (input) => waitForOpenLiveEvalInputRequest(context, input),
+        submitInputResponse: (input) => submitLiveEvalInputResponse(context, input),
+        cancelInputRequest: (input) => cancelLiveEvalInputRequest(context, input),
+    };
+}
+/** Create live eval conversation. */
+export async function createLiveEvalConversation(context, input) {
+    const response = await createFetch(context)(createApiUrl(context, "/conversations"), {
+        method: "POST",
+        headers: createLiveEvalJsonHeaders(context),
+        body: JSON.stringify({
+            ...(context.projectId ? { project_id: context.projectId } : {}),
+            title: input.title,
+        }),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to create eval conversation: ${response.status} ${await getResponseText(response)}`);
+    }
+    const payload = getLiveEvalIdResponseSchema().parse(await response.json());
+    if (!payload.id) {
+        throw new Error("Conversation creation response did not include id");
+    }
+    return payload.id;
+}
+/** Delete live eval conversation helper. */
+export async function deleteLiveEvalConversation(context, input) {
+    const response = await createFetch(context)(createApiUrl(context, `/conversations/${input.conversationId}`), {
+        method: "DELETE",
+        headers: createLiveEvalAuthHeaders(context),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok && response.status !== 404) {
+        throw new Error(`Failed to delete eval conversation ${input.conversationId}: ${response.status} ${await getResponseText(response)}`);
+    }
+}
+/** Create live eval project upload fixture. */
+export async function createLiveEvalProjectUploadFixture(context, input) {
+    const projectId = requireLiveEvalProjectId(context.projectId, "Project upload fixtures require a live-eval project id");
+    const createResponse = await createFetch(context)(createApiUrl(context, `/projects/${projectId}/uploads`), {
+        method: "POST",
+        headers: createLiveEvalJsonHeaders(context),
+        body: JSON.stringify({
+            file_path: input.filePath,
+            content_type: input.contentType,
+            size: getProjectUploadBodySize(input.body, input.size),
+        }),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!createResponse.ok) {
+        throw new Error(`Failed to create project upload URL: ${createResponse.status} ${await getResponseText(createResponse)}`);
+    }
+    const createPayload = getProjectUploadResponseSchema().parse(await createResponse.json());
+    if (!createPayload.file_upload_url) {
+        throw new Error("Project upload response did not include file_upload_url");
+    }
+    const uploadResponse = await createFetch(context)(createPayload.file_upload_url, {
+        method: "PUT",
+        headers: createProjectUploadHeaders(createPayload.required_headers, input.contentType),
+        body: createProjectUploadBody(input.body, input.contentType),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!uploadResponse.ok) {
+        throw new Error(`Failed to upload project fixture: ${uploadResponse.status} ${await getResponseText(uploadResponse)}`);
+    }
+    return waitForProjectUploadFixture(context, {
+        projectId,
+        filePath: input.filePath,
+        requestTimeoutMs: input.requestTimeoutMs,
+        pollIntervalMs: input.pollIntervalMs,
+        maxAttempts: input.maxAttempts,
+    });
+}
+/** Return live eval project file. */
+export async function getLiveEvalProjectFile(context, input) {
+    const projectId = requireLiveEvalProjectId(context.projectId, "getLiveEvalProjectFile requires a live-eval project id");
+    const response = await createFetch(context)(createApiUrl(context, `/projects/${projectId}/files/${encodeURIComponent(input.filePath)}`), {
+        headers: createLiveEvalAuthHeaders(context),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (response.status === 404) {
+        return null;
+    }
+    if (!response.ok) {
+        throw new Error(`Failed to read project file: ${response.status} ${await getResponseText(response)}`);
+    }
+    const payload = getProjectFileResponseSchema().parse(await response.json());
+    return {
+        path: payload.path ?? input.filePath,
+        content: payload.content ?? "",
+    };
+}
+/** Create live eval release. */
+export async function createLiveEvalRelease(context, input) {
+    const projectId = requireLiveEvalProjectId(context.projectId, "createLiveEvalRelease requires a live-eval project id");
+    const response = await createFetch(context)(createApiUrl(context, `/projects/${projectId}/releases`), {
+        method: "POST",
+        headers: createLiveEvalJsonHeaders(context),
+        body: JSON.stringify({
+            description: input.description ?? "eval platform capability release",
+        }),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to create release: ${response.status} ${await getResponseText(response)}`);
+    }
+    const payload = getLiveEvalIdResponseSchema().parse(await response.json());
+    if (!payload.id) {
+        throw new Error("Release creation response did not include id");
+    }
+    return payload.id;
+}
+/** Delete live eval project file helper. */
+export async function deleteLiveEvalProjectFile(context, input) {
+    const projectId = context.projectId;
+    if (!projectId) {
+        return;
+    }
+    const response = await createFetch(context)(createApiUrl(context, `/projects/${projectId}/files/${encodeURIComponent(input.filePath)}`), {
+        method: "DELETE",
+        headers: createLiveEvalAuthHeaders(context),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok && response.status !== 404) {
+        throw new Error(`Failed to delete project file: ${response.status} ${await getResponseText(response)}`);
+    }
+}
+/** List open live eval input requests. */
+export async function listOpenLiveEvalInputRequests(context, input) {
+    const response = await createFetch(context)(createApiUrl(context, `/conversations/${input.conversationId}/input-requests?status=open`), {
+        headers: createLiveEvalAuthHeaders(context),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to list eval input requests: ${response.status} ${await getResponseText(response)}`);
+    }
+    const payload = getInputRequestListResponseSchema().parse(await response.json());
+    return (payload.data ?? []).flatMap((item) => {
+        const parsed = getInputRequestRecordSchema().safeParse(item);
+        return parsed.success ? [parsed.data] : [];
+    });
+}
+/** Request payload for wait for open live eval input. */
+export async function waitForOpenLiveEvalInputRequest(context, input) {
+    const timeoutMs = input.timeoutMs ?? 30_000;
+    const pollIntervalMs = input.pollIntervalMs ?? 500;
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        if (input.abortSignal.aborted) {
+            throw new Error("Eval sidecar aborted before an input request appeared");
+        }
+        const requests = await listOpenLiveEvalInputRequests(context, input);
+        const request = requests[0];
+        if (request) {
+            return request.id;
+        }
+        await wait({ ms: pollIntervalMs });
+    }
+    throw new Error(`Timed out while waiting for an open input request in conversation ${input.conversationId}`);
+}
+/** Response payload for submit live eval input. */
+export async function submitLiveEvalInputResponse(context, input) {
+    const response = await createFetch(context)(createApiUrl(context, `/conversations/${input.conversationId}/input-requests/${input.inputRequestId}/responses`), {
+        method: "POST",
+        headers: createLiveEvalJsonHeaders(context),
+        body: JSON.stringify({ values: input.values }),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to submit eval input response: ${response.status} ${await getResponseText(response)}`);
+    }
+}
+/** Request payload for cancel live eval input. */
+export async function cancelLiveEvalInputRequest(context, input) {
+    const response = await createFetch(context)(createApiUrl(context, `/conversations/${input.conversationId}/input-requests/${input.inputRequestId}/cancel`), {
+        method: "POST",
+        headers: createLiveEvalAuthHeaders(context),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to cancel eval input request: ${response.status} ${await getResponseText(response)}`);
+    }
+}

package/esm/src/agent/testing/live-evals/cli-runner.d.ts

@@ -0,0 +1,40 @@
+import { type LiveEvalRuntime } from "./performance.js";
+import { containsSkillLoad, countStepStartedEvents, createLiveEvalCaseSupport, hasFinished, type LiveEvalCase, type LiveEvalRunnerConfig } from "./runner.js";
+type EnvRecord = Record<string, string | undefined>;
+/** Public API contract for live eval cli case groups. */
+export interface LiveEvalCliCaseGroups {
+    readOnlyCases: LiveEvalCase[];
+    writeCases: LiveEvalCase[];
+    experimentalWriteCases: LiveEvalCase[];
+}
+/** Input payload for live eval cli case factory. */
+export interface LiveEvalCliCaseFactoryInput {
+    authToken: string;
+    endpoint: string;
+    projectId: string | null;
+    branchId: string | null;
+    model: string | null;
+    requestTimeoutMs: number;
+    enableLlmJudge: boolean;
+    hasFinished: typeof hasFinished;
+    containsSkillLoad: typeof containsSkillLoad;
+    countStepStartedEvents: typeof countStepStartedEvents;
+    verifyFileExists: ReturnType<typeof createLiveEvalCaseSupport>["verifyFileExists"];
+    withJudge: ReturnType<typeof createLiveEvalCaseSupport>["withJudge"];
+    judgeLlm: ReturnType<typeof createLiveEvalCaseSupport>["judgeLlm"];
+}
+/** Input payload for run live eval cli. */
+export interface RunLiveEvalCliInput {
+    env: EnvRecord;
+    caseSets: Record<string, readonly string[]>;
+    createCases: (input: LiveEvalCliCaseFactoryInput) => LiveEvalCliCaseGroups;
+    runtimes?: readonly LiveEvalRuntime[];
+    cwd?: string;
+    log?: (message: string) => void;
+    error?: (message: string) => void;
+    createCaseSupport?: (config: LiveEvalRunnerConfig) => ReturnType<typeof createLiveEvalCaseSupport>;
+}
+/** Run live eval cli. */
+export declare function runLiveEvalCli(input: RunLiveEvalCliInput): Promise<number>;
+export {};
+//# sourceMappingURL=cli-runner.d.ts.map

package/esm/src/agent/testing/live-evals/cli-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-runner.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/cli-runner.ts"],"names":[],"mappings":"AAGA,OAAO,EAAkC,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAQxF,OAAO,EACL,iBAAiB,EACjB,sBAAsB,EACtB,yBAAyB,EACzB,WAAW,EACX,KAAK,YAAY,EACjB,KAAK,oBAAoB,EAC1B,MAAM,aAAa,CAAC;AAKrB,KAAK,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;AAEpD,yDAAyD;AACzD,MAAM,WAAW,qBAAqB;IACpC,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,UAAU,EAAE,YAAY,EAAE,CAAC;IAC3B,sBAAsB,EAAE,YAAY,EAAE,CAAC;CACxC;AAED,oDAAoD;AACpD,MAAM,WAAW,2BAA2B;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,OAAO,CAAC;IACxB,WAAW,EAAE,OAAO,WAAW,CAAC;IAChC,iBAAiB,EAAE,OAAO,iBAAiB,CAAC;IAC5C,sBAAsB,EAAE,OAAO,sBAAsB,CAAC;IACtD,gBAAgB,EAAE,UAAU,CAAC,OAAO,yBAAyB,CAAC,CAAC,kBAAkB,CAAC,CAAC;IACnF,SAAS,EAAE,UAAU,CAAC,OAAO,yBAAyB,CAAC,CAAC,WAAW,CAAC,CAAC;IACrE,QAAQ,EAAE,UAAU,CAAC,OAAO,yBAAyB,CAAC,CAAC,UAAU,CAAC,CAAC;CACpE;AAED,2CAA2C;AAC3C,MAAM,WAAW,mBAAmB;IAClC,GAAG,EAAE,SAAS,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;IAC5C,WAAW,EAAE,CAAC,KAAK,EAAE,2BAA2B,KAAK,qBAAqB,CAAC;IAC3E,QAAQ,CAAC,EAAE,SAAS,eAAe,EAAE,CAAC;IACtC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,iBAAiB,CAAC,EAAE,CAClB,MAAM,EAAE,oBAAoB,KACzB,UAAU,CAAC,OAAO,yBAAyB,CAAC,CAAC;CACnD;AAuBD,yBAAyB;AACzB,wBAAsB,cAAc,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,CAwJhF"}

package/esm/src/agent/testing/live-evals/cli-runner.js

@@ -0,0 +1,144 @@
+import { mkdir, writeTextFile } from "../../../platform/compat/fs.js";
+import { dirname, resolve } from "../../../platform/compat/path/index.js";
+import { cwd as getProcessCwd } from "../../../platform/compat/process.js";
+import { buildRuntimePerformanceSummary } from "./performance.js";
+import { buildLiveEvalCaseTagSummary, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, resolveLiveEvalRequestedCaseIds, selectLiveEvalCases, } from "./report.js";
+import { containsSkillLoad, countStepStartedEvents, createLiveEvalCaseSupport, hasFinished, } from "./runner.js";
+import { getLiveEvalProjectFile } from "./api-client.js";
+import { resolveLiveEvalEnvironment } from "./environment.js";
+function splitCsvEnv(value) {
+    return new Set((value ?? "")
+        .split(",")
+        .map((entry) => entry.trim())
+        .filter((entry) => entry.length > 0));
+}
+function createTimestampedReportPath(input) {
+    return resolve(input.cwd, ".omx/logs", input.directory, `${new Date().toISOString().replaceAll(":", "-").replaceAll(".", "-")}.json`);
+}
+/** Run live eval cli. */
+export async function runLiveEvalCli(input) {
+    const log = input.log ?? console.log;
+    const error = input.error ?? console.error;
+    const cwd = input.cwd ?? getProcessCwd();
+    const { endpoint, authToken, apiUrl, projectId, branchId, model } = resolveLiveEvalEnvironment(input.env);
+    const requestedRuntimeSelection = input.runtimes ?? ["framework"];
+    const runWriteEvals = input.env.AG_UI_EVAL_WRITE === "1";
+    const runExperimentalWriteEvals = input.env.AG_UI_EVAL_EXPERIMENTAL === "1";
+    const requestTimeoutMs = Number(input.env.AG_UI_EVAL_TIMEOUT_MS ?? "240000");
+    const progressLogIntervalMs = Number(input.env.AG_UI_EVAL_PROGRESS_MS ?? "15000");
+    const reportPath = input.env.AG_UI_EVAL_REPORT_PATH ??
+        createTimestampedReportPath({ cwd, directory: "ag-ui-live-evals" });
+    const requestedCaseIds = splitCsvEnv(input.env.AG_UI_EVAL_CASES);
+    const requestedCaseTags = splitCsvEnv(input.env.AG_UI_EVAL_TAGS);
+    const requestedCaseSetId = input.env.AG_UI_EVAL_CASE_SET?.trim() || null;
+    const enableLlmJudge = input.env.AG_UI_EVAL_LLM_JUDGE === "1";
+    const apiContext = {
+        apiUrl,
+        authToken,
+        projectId: projectId ?? null,
+    };
+    const createCaseSupport = input.createCaseSupport ?? createLiveEvalCaseSupport;
+    const { judgeLlm, runEval, verifyFileExists, withJudge } = createCaseSupport({
+        endpoint,
+        authToken,
+        apiUrl,
+        projectId: projectId ?? null,
+        branchId: branchId ?? null,
+        model: model ?? null,
+        requestTimeoutMs,
+        progressLogIntervalMs,
+        enableLlmJudge,
+        readProjectFile: (readerInput) => getLiveEvalProjectFile(apiContext, readerInput),
+    });
+    const { readOnlyCases, writeCases, experimentalWriteCases } = input.createCases({
+        authToken,
+        endpoint,
+        projectId: projectId ?? null,
+        branchId: branchId ?? null,
+        model: model ?? null,
+        requestTimeoutMs,
+        enableLlmJudge,
+        hasFinished,
+        containsSkillLoad,
+        countStepStartedEvents,
+        verifyFileExists,
+        withJudge,
+        judgeLlm,
+    });
+    if (authToken.length === 0) {
+        error("Missing VERYFRONT_TOKEN");
+        return 1;
+    }
+    log(`AG-UI live evals -> ${endpoint}`);
+    log(`Veryfront API -> ${apiUrl}`);
+    log(`Project scope -> ${projectId ?? "none"}`);
+    log(`Runtime -> ${requestedRuntimeSelection.join(", ")}`);
+    log(`Write evals -> ${runWriteEvals ? "enabled" : "disabled"}`);
+    log(`Experimental evals -> ${runExperimentalWriteEvals ? "enabled" : "disabled"}`);
+    log(`Case set -> ${requestedCaseSetId ?? "none"}`);
+    log(`Case tags -> ${requestedCaseTags.size > 0 ? [...requestedCaseTags].join(", ") : "none"}`);
+    const allCases = [...readOnlyCases, ...writeCases, ...experimentalWriteCases];
+    const resolvedRequestedCaseIds = resolveLiveEvalRequestedCaseIds({
+        caseSets: input.caseSets,
+        requestedCaseIds,
+        requestedCaseSetId,
+    });
+    const cases = selectLiveEvalCases({
+        allCases,
+        readOnlyCases,
+        writeCases,
+        experimentalWriteCases,
+        requestedCaseIds: resolvedRequestedCaseIds,
+        requestedCaseTags,
+        runWriteEvals,
+        runExperimentalWriteEvals,
+    });
+    const selectedCaseTagSummary = buildLiveEvalCaseTagSummary(cases);
+    if (cases.length === 0) {
+        error("No eval cases selected.");
+        return 1;
+    }
+    const results = [];
+    for (const runtime of requestedRuntimeSelection) {
+        log(`\n[runtime] ${runtime}`);
+        for (const testCase of cases) {
+            log(`\n[run] ${runtime} :: ${testCase.label}`);
+            const result = await runEval(testCase, runtime);
+            results.push(result);
+            log(`[${runtime}] [${result.status}] ${result.details}`);
+        }
+    }
+    const summary = buildLiveEvalStatusSummary(results);
+    const runtimeSummary = buildLiveEvalRuntimeSummary(requestedRuntimeSelection, results);
+    const runtimePerformanceSummary = buildRuntimePerformanceSummary(results);
+    log("\nSummary");
+    log(`passed: ${summary.passed}`);
+    log(`failed: ${summary.failed}`);
+    log(`skipped: ${summary.skipped}`);
+    for (const runtime of requestedRuntimeSelection) {
+        const currentRuntimeSummary = runtimeSummary[runtime];
+        log(`${runtime}: passed=${currentRuntimeSummary.passed} failed=${currentRuntimeSummary.failed} skipped=${currentRuntimeSummary.skipped}`);
+        const performance = runtimePerformanceSummary[runtime];
+        log(`${runtime}: avg=${performance.avgDurationMs}ms p50=${performance.p50DurationMs}ms p95=${performance.p95DurationMs}ms min=${performance.minDurationMs}ms max=${performance.maxDurationMs}ms`);
+    }
+    await mkdir(dirname(reportPath), { recursive: true });
+    await writeTextFile(reportPath, JSON.stringify({
+        generatedAt: new Date().toISOString(),
+        endpoint,
+        apiUrl,
+        projectId: projectId ?? null,
+        runtimes: requestedRuntimeSelection,
+        writeEvals: runWriteEvals,
+        requestedCaseIds: [...resolvedRequestedCaseIds],
+        requestedCaseTags: [...requestedCaseTags],
+        requestedCaseSetId,
+        caseMetadata: Object.fromEntries(cases.map((testCase) => [testCase.id, testCase.metadata ?? { tags: [] }])),
+        selectedCaseTagSummary,
+        results,
+        summary,
+        runtimeSummary,
+        runtimePerformanceSummary,
+    }, null, 2));
+    log(`report: ${reportPath}`);
+    return summary.failed > 0 ? 1 : 0;
+}

package/esm/src/agent/testing/live-evals/environment.d.ts

@@ -0,0 +1,15 @@
+import { type AgentServiceConfigInput } from "../../service/config.js";
+/** Public API contract for live eval environment. */
+export interface LiveEvalEnvironment {
+    endpoint: string;
+    authToken: string;
+    apiUrl: string;
+    projectId: string | undefined;
+    branchId: string | undefined;
+    model: string | undefined;
+}
+/** Default value for live eval endpoint. */
+export declare const DEFAULT_LIVE_EVAL_ENDPOINT = "http://127.0.0.1:3001/api/ag-ui";
+/** Resolves live eval environment. */
+export declare function resolveLiveEvalEnvironment(env?: AgentServiceConfigInput): LiveEvalEnvironment;
+//# sourceMappingURL=environment.d.ts.map

package/esm/src/agent/testing/live-evals/environment.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"environment.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/environment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,uBAAuB,EAA2B,MAAM,yBAAyB,CAAC;AAEhG,qDAAqD;AACrD,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3B;AAED,4CAA4C;AAC5C,eAAO,MAAM,0BAA0B,oCAAoC,CAAC;AAE5E,sCAAsC;AACtC,wBAAgB,0BAA0B,CACxC,GAAG,GAAE,uBAA4B,GAChC,mBAAmB,CAerB"}

package/esm/src/agent/testing/live-evals/environment.js

@@ -0,0 +1,20 @@
+import { parseAgentServiceConfig } from "../../service/config.js";
+/** Default value for live eval endpoint. */
+export const DEFAULT_LIVE_EVAL_ENDPOINT = "http://127.0.0.1:3001/api/ag-ui";
+/** Resolves live eval environment. */
+export function resolveLiveEvalEnvironment(env = {}) {
+    return {
+        endpoint: typeof env.AG_UI_EVAL_ENDPOINT === "string"
+            ? env.AG_UI_EVAL_ENDPOINT
+            : DEFAULT_LIVE_EVAL_ENDPOINT,
+        authToken: typeof env.VERYFRONT_TOKEN === "string" ? env.VERYFRONT_TOKEN : "",
+        apiUrl: typeof env.VERYFRONT_API_URL === "string"
+            ? env.VERYFRONT_API_URL
+            : parseAgentServiceConfig(env).VERYFRONT_API_URL,
+        projectId: typeof env.AG_UI_EVAL_PROJECT_ID === "string"
+            ? env.AG_UI_EVAL_PROJECT_ID
+            : undefined,
+        branchId: typeof env.AG_UI_EVAL_BRANCH_ID === "string" ? env.AG_UI_EVAL_BRANCH_ID : undefined,
+        model: typeof env.AG_UI_EVAL_MODEL === "string" ? env.AG_UI_EVAL_MODEL : undefined,
+    };
+}

package/esm/src/agent/testing/live-evals/formatting.d.ts

@@ -0,0 +1,15 @@
+import { Buffer } from "node:buffer";
+import type { AgUiSseProgressSnapshot as EvalProgressSnapshot } from "../../index.js";
+/** Create plain text pdf. */
+export declare function createPlainTextPdf(lines: string[]): Buffer;
+/** Builds progress line. */
+export declare function buildProgressLine(input: {
+    caseId: string;
+    startedAt: number;
+    progress: EvalProgressSnapshot;
+}): string;
+/** Builds failure suffix. */
+export declare function buildFailureSuffix(progress: EvalProgressSnapshot): string;
+/** Contains ordered subsequence helper. */
+export declare function containsOrderedSubsequence(haystack: string[], needle: string[]): boolean;
+//# sourceMappingURL=formatting.d.ts.map

package/esm/src/agent/testing/live-evals/formatting.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"formatting.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/formatting.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,KAAK,EAAE,uBAAuB,IAAI,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAStF,6BAA6B;AAC7B,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CA4C1D;AAED,4BAA4B;AAC5B,wBAAgB,iBAAiB,CAAC,KAAK,EAAE;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,oBAAoB,CAAC;CAChC,GAAG,MAAM,CAcT;AAED,6BAA6B;AAC7B,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,oBAAoB,GAAG,MAAM,CAUzE;AAED,2CAA2C;AAC3C,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,MAAM,EAAE,EAClB,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAcT"}

package/esm/src/agent/testing/live-evals/formatting.js

@@ -0,0 +1,75 @@
+import { Buffer } from "node:buffer";
+function escapePdfText(input) {
+    return input.replaceAll("\\", "\\\\").replaceAll("(", "\\(").replaceAll(")", "\\)");
+}
+/** Create plain text pdf. */
+export function createPlainTextPdf(lines) {
+    const contentLines = [
+        "BT",
+        "/F1 18 Tf",
+        "72 720 Td",
+        `(${escapePdfText(lines[0] ?? "Untitled")}) Tj`,
+        "/F1 12 Tf",
+        ...lines.slice(1).flatMap((line) => ["0 -20 Td", `(${escapePdfText(line)}) Tj`]),
+        "ET",
+    ];
+    const contentStream = contentLines.join("\n");
+    const objects = [
+        "<< /Type /Catalog /Pages 2 0 R >>",
+        "<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+        "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>",
+        `<< /Length ${Buffer.byteLength(contentStream, "utf8")} >>\nstream\n${contentStream}\nendstream`,
+        "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+    ];
+    let pdf = "%PDF-1.4\n";
+    const offsets = [0];
+    objects.forEach((objectBody, index) => {
+        offsets.push(Buffer.byteLength(pdf, "utf8"));
+        pdf += `${index + 1} 0 obj\n${objectBody}\nendobj\n`;
+    });
+    const xrefOffset = Buffer.byteLength(pdf, "utf8");
+    pdf += `xref\n0 ${objects.length + 1}\n`;
+    pdf += "0000000000 65535 f \n";
+    offsets.slice(1).forEach((offset) => {
+        pdf += `${offset.toString().padStart(10, "0")} 00000 n \n`;
+    });
+    pdf += `trailer\n<< /Root 1 0 R /Size ${objects.length + 1} >>\nstartxref\n${xrefOffset}\n%%EOF\n`;
+    return Buffer.from(pdf, "utf8");
+}
+/** Builds progress line. */
+export function buildProgressLine(input) {
+    const elapsedSeconds = Math.max(1, Math.round((Date.now() - input.startedAt) / 1000));
+    const lastEvent = input.progress.lastEventType ?? "none";
+    const lastTool = input.progress.lastToolCallName
+        ? ` tool=${input.progress.lastToolCallName}`
+        : "";
+    const toolSummary = input.progress.toolStarts.length > 0
+        ? ` tools=${input.progress.toolStarts.join(",")}`
+        : "";
+    const textSummary = input.progress.textLength > 0 ? ` text=${input.progress.textLength}ch` : "";
+    return `[progress] ${input.caseId} ${elapsedSeconds}s events=${input.progress.eventCount} last=${lastEvent}${lastTool}${toolSummary}${textSummary}`;
+}
+/** Builds failure suffix. */
+export function buildFailureSuffix(progress) {
+    const details = [
+        `events=${progress.eventCount}`,
+        `last=${progress.lastEventType ?? "none"}`,
+        progress.lastToolCallName ? `tool=${progress.lastToolCallName}` : null,
+        progress.toolStarts.length > 0 ? `tools=${progress.toolStarts.join(",")}` : null,
+        progress.textLength > 0 ? `text=${progress.textLength}ch` : null,
+    ].filter((value) => value !== null);
+    return details.length > 0 ? ` Progress: ${details.join(" ")}` : "";
+}
+/** Contains ordered subsequence helper. */
+export function containsOrderedSubsequence(haystack, needle) {
+    let cursor = 0;
+    for (const value of haystack) {
+        if (value === needle[cursor]) {
+            cursor += 1;
+        }
+        if (cursor === needle.length) {
+            return true;
+        }
+    }
+    return cursor === needle.length;
+}

package/esm/src/agent/testing/live-evals/index.d.ts

@@ -0,0 +1,12 @@
+export { type LiveEvalCliCaseFactoryInput, type LiveEvalCliCaseGroups, runLiveEvalCli, type RunLiveEvalCliInput, } from "./cli-runner.js";
+export { DEFAULT_LIVE_EVAL_ENDPOINT, type LiveEvalEnvironment, resolveLiveEvalEnvironment, } from "./environment.js";
+export { cancelLiveEvalInputRequest, createLiveEvalApiClient, createLiveEvalConversation, createLiveEvalProjectUploadFixture, createLiveEvalRelease, deleteLiveEvalConversation, deleteLiveEvalProjectFile, getLiveEvalProjectFile, listOpenLiveEvalInputRequests, type LiveEvalApiClient, type LiveEvalApiContext, type LiveEvalConversationInput, type LiveEvalCreateConversationInput, type LiveEvalCreateReleaseInput, type LiveEvalInputRequestInput, type LiveEvalInputRequestRecord, type LiveEvalInputResponseValues, type LiveEvalProjectFileInput, type LiveEvalProjectUploadFixtureInput, type LiveEvalRequestTimeoutInput, type LiveEvalSubmitInputResponseInput, type LiveEvalWaitForOpenInputRequestInput, submitLiveEvalInputResponse, waitForOpenLiveEvalInputRequest, } from "./api-client.js";
+export { buildFailureSuffix, buildProgressLine, containsOrderedSubsequence, createPlainTextPdf, } from "./formatting.js";
+export { evaluateRuntimeConfidenceEnv, printRuntimeConfidencePreflight, type RuntimeConfidencePreflightResult, } from "./preflight.js";
+export { buildRuntimePerformanceSummary, type LiveEvalResultForPerformance, type LiveEvalRuntime, type RuntimePerformanceSummary, } from "./performance.js";
+export { buildLiveEvalCaseMetadata, type BuildLiveEvalCaseMetadataInput, DEFAULT_LIVE_EVAL_AREA_TAG_RULES, DEFAULT_LIVE_EVAL_OPTIONAL_JUDGE_CASE_PREFIXES, type LiveEvalCaseMetadataOptions, type LiveEvalCaseSurface, type LiveEvalCaseTagRule, withLiveEvalMetadata, } from "./metadata.js";
+export { buildLiveEvalRequestBody, type BuildLiveEvalRequestBodyInput, type LiveEvalRequestBody, } from "./request.js";
+export { buildLiveEvalCaseTagSummary, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, hasEveryLiveEvalTag, type LiveEvalCaseMetadata, type LiveEvalCaseSelectionInput, type LiveEvalResultForReport, resolveLiveEvalRequestedCaseIds, selectLiveEvalCases, } from "./report.js";
+export { createFailedEvalResult, createPassedEvalResult, createSkippedEvalResult, type LiveEvalResultRecord, } from "./result.js";
+export { containsSkillLoad, countStepStartedEvents, createLiveEvalCaseSupport, hasFinished, type LiveEvalCase, type LiveEvalContext, type LiveEvalProjectFile, type LiveEvalProjectFileReaderInput, type LiveEvalRunnerConfig, liveEvalRunnerInternals, type PreparedLiveEvalInput, } from "./runner.js";
+//# sourceMappingURL=index.d.ts.map

package/esm/src/agent/testing/live-evals/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,2BAA2B,EAChC,KAAK,qBAAqB,EAC1B,cAAc,EACd,KAAK,mBAAmB,GACzB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,0BAA0B,EAC1B,KAAK,mBAAmB,EACxB,0BAA0B,GAC3B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,0BAA0B,EAC1B,uBAAuB,EACvB,0BAA0B,EAC1B,kCAAkC,EAClC,qBAAqB,EACrB,0BAA0B,EAC1B,yBAAyB,EACzB,sBAAsB,EACtB,6BAA6B,EAC7B,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,yBAAyB,EAC9B,KAAK,+BAA+B,EACpC,KAAK,0BAA0B,EAC/B,KAAK,yBAAyB,EAC9B,KAAK,0BAA0B,EAC/B,KAAK,2BAA2B,EAChC,KAAK,wBAAwB,EAC7B,KAAK,iCAAiC,EACtC,KAAK,2BAA2B,EAChC,KAAK,gCAAgC,EACrC,KAAK,oCAAoC,EACzC,2BAA2B,EAC3B,+BAA+B,GAChC,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,0BAA0B,EAC1B,kBAAkB,GACnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,4BAA4B,EAC5B,+BAA+B,EAC/B,KAAK,gCAAgC,GACtC,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,8BAA8B,EAC9B,KAAK,4BAA4B,EACjC,KAAK,eAAe,EACpB,KAAK,yBAAyB,GAC/B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,yBAAyB,EACzB,KAAK,8BAA8B,EACnC,gCAAgC,EAChC,8CAA8C,EAC9C,KAAK,2BAA2B,EAChC,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,oBAAoB,GACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,wBAAwB,EACxB,KAAK,6BAA6B,EAClC,KAAK,mBAAmB,GACzB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,2BAA2B,EAC3B,2BAA2B,EAC3B,0BAA0B,EAC1B,mBAAmB,EACnB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,EAC/B,KAAK,uBAAuB,EAC5B,+BAA+B,EAC/B,mBAAmB,GACpB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,KAAK,oBAAoB,GAC1B,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,iBAAiB,EACjB,sBAAsB,EACtB,yBAAyB,EACzB,WAAW,EACX,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,8BAA8B,EACnC,KAAK,oBAAoB,EACzB,uBAAuB,EACvB,KAAK,qBAAqB,GAC3B,MAAM,aAAa,CAAC"}