veryfront 0.1.877 → 0.1.878

package/esm/src/eval/agent-service/live-evals/cli-runner.js

@@ -0,0 +1,144 @@
+import { mkdir, writeTextFile } from "../../../platform/compat/fs.js";
+import { dirname, resolve } from "../../../platform/compat/path/index.js";
+import { cwd as getProcessCwd } from "../../../platform/compat/process.js";
+import { buildRuntimePerformanceSummary } from "./performance.js";
+import { buildLiveEvalCaseTagSummary, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, resolveLiveEvalRequestedCaseIds, selectLiveEvalCases, } from "./report.js";
+import { containsSkillLoad, countStepStartedEvents, createLiveEvalCaseSupport, hasFinished, } from "./runner.js";
+import { getLiveEvalProjectFile } from "./api-client.js";
+import { resolveLiveEvalEnvironment } from "./environment.js";
+function splitCsvEnv(value) {
+    return new Set((value ?? "")
+        .split(",")
+        .map((entry) => entry.trim())
+        .filter((entry) => entry.length > 0));
+}
+function createTimestampedReportPath(input) {
+    return resolve(input.cwd, ".omx/logs", input.directory, `${new Date().toISOString().replaceAll(":", "-").replaceAll(".", "-")}.json`);
+}
+/** Run live eval cli. */
+export async function runLiveEvalCli(input) {
+    const log = input.log ?? console.log;
+    const error = input.error ?? console.error;
+    const cwd = input.cwd ?? getProcessCwd();
+    const { endpoint, authToken, apiUrl, projectId, branchId, model } = resolveLiveEvalEnvironment(input.env);
+    const requestedRuntimeSelection = input.runtimes ?? ["framework"];
+    const runWriteEvals = input.env.AG_UI_EVAL_WRITE === "1";
+    const runExperimentalWriteEvals = input.env.AG_UI_EVAL_EXPERIMENTAL === "1";
+    const requestTimeoutMs = Number(input.env.AG_UI_EVAL_TIMEOUT_MS ?? "240000");
+    const progressLogIntervalMs = Number(input.env.AG_UI_EVAL_PROGRESS_MS ?? "15000");
+    const reportPath = input.env.AG_UI_EVAL_REPORT_PATH ??
+        createTimestampedReportPath({ cwd, directory: "ag-ui-live-evals" });
+    const requestedCaseIds = splitCsvEnv(input.env.AG_UI_EVAL_CASES);
+    const requestedCaseTags = splitCsvEnv(input.env.AG_UI_EVAL_TAGS);
+    const requestedCaseSetId = input.env.AG_UI_EVAL_CASE_SET?.trim() || null;
+    const enableLlmJudge = input.env.AG_UI_EVAL_LLM_JUDGE === "1";
+    const apiContext = {
+        apiUrl,
+        authToken,
+        projectId: projectId ?? null,
+    };
+    const createCaseSupport = input.createCaseSupport ?? createLiveEvalCaseSupport;
+    const { judgeLlm, runEval, verifyFileExists, withJudge } = createCaseSupport({
+        endpoint,
+        authToken,
+        apiUrl,
+        projectId: projectId ?? null,
+        branchId: branchId ?? null,
+        model: model ?? null,
+        requestTimeoutMs,
+        progressLogIntervalMs,
+        enableLlmJudge,
+        readProjectFile: (readerInput) => getLiveEvalProjectFile(apiContext, readerInput),
+    });
+    const { readOnlyCases, writeCases, experimentalWriteCases } = input.createCases({
+        authToken,
+        endpoint,
+        projectId: projectId ?? null,
+        branchId: branchId ?? null,
+        model: model ?? null,
+        requestTimeoutMs,
+        enableLlmJudge,
+        hasFinished,
+        containsSkillLoad,
+        countStepStartedEvents,
+        verifyFileExists,
+        withJudge,
+        judgeLlm,
+    });
+    if (authToken.length === 0) {
+        error("Missing VERYFRONT_TOKEN");
+        return 1;
+    }
+    log(`AG-UI live evals -> ${endpoint}`);
+    log(`Veryfront API -> ${apiUrl}`);
+    log(`Project scope -> ${projectId ?? "none"}`);
+    log(`Runtime -> ${requestedRuntimeSelection.join(", ")}`);
+    log(`Write evals -> ${runWriteEvals ? "enabled" : "disabled"}`);
+    log(`Experimental evals -> ${runExperimentalWriteEvals ? "enabled" : "disabled"}`);
+    log(`Case set -> ${requestedCaseSetId ?? "none"}`);
+    log(`Case tags -> ${requestedCaseTags.size > 0 ? [...requestedCaseTags].join(", ") : "none"}`);
+    const allCases = [...readOnlyCases, ...writeCases, ...experimentalWriteCases];
+    const resolvedRequestedCaseIds = resolveLiveEvalRequestedCaseIds({
+        caseSets: input.caseSets,
+        requestedCaseIds,
+        requestedCaseSetId,
+    });
+    const cases = selectLiveEvalCases({
+        allCases,
+        readOnlyCases,
+        writeCases,
+        experimentalWriteCases,
+        requestedCaseIds: resolvedRequestedCaseIds,
+        requestedCaseTags,
+        runWriteEvals,
+        runExperimentalWriteEvals,
+    });
+    const selectedCaseTagSummary = buildLiveEvalCaseTagSummary(cases);
+    if (cases.length === 0) {
+        error("No eval cases selected.");
+        return 1;
+    }
+    const results = [];
+    for (const runtime of requestedRuntimeSelection) {
+        log(`\n[runtime] ${runtime}`);
+        for (const testCase of cases) {
+            log(`\n[run] ${runtime} :: ${testCase.label}`);
+            const result = await runEval(testCase, runtime);
+            results.push(result);
+            log(`[${runtime}] [${result.status}] ${result.details}`);
+        }
+    }
+    const summary = buildLiveEvalStatusSummary(results);
+    const runtimeSummary = buildLiveEvalRuntimeSummary(requestedRuntimeSelection, results);
+    const runtimePerformanceSummary = buildRuntimePerformanceSummary(results);
+    log("\nSummary");
+    log(`passed: ${summary.passed}`);
+    log(`failed: ${summary.failed}`);
+    log(`skipped: ${summary.skipped}`);
+    for (const runtime of requestedRuntimeSelection) {
+        const currentRuntimeSummary = runtimeSummary[runtime];
+        log(`${runtime}: passed=${currentRuntimeSummary.passed} failed=${currentRuntimeSummary.failed} skipped=${currentRuntimeSummary.skipped}`);
+        const performance = runtimePerformanceSummary[runtime];
+        log(`${runtime}: avg=${performance.avgDurationMs}ms p50=${performance.p50DurationMs}ms p95=${performance.p95DurationMs}ms min=${performance.minDurationMs}ms max=${performance.maxDurationMs}ms`);
+    }
+    await mkdir(dirname(reportPath), { recursive: true });
+    await writeTextFile(reportPath, JSON.stringify({
+        generatedAt: new Date().toISOString(),
+        endpoint,
+        apiUrl,
+        projectId: projectId ?? null,
+        runtimes: requestedRuntimeSelection,
+        writeEvals: runWriteEvals,
+        requestedCaseIds: [...resolvedRequestedCaseIds],
+        requestedCaseTags: [...requestedCaseTags],
+        requestedCaseSetId,
+        caseMetadata: Object.fromEntries(cases.map((testCase) => [testCase.id, testCase.metadata ?? { tags: [] }])),
+        selectedCaseTagSummary,
+        results,
+        summary,
+        runtimeSummary,
+        runtimePerformanceSummary,
+    }, null, 2));
+    log(`report: ${reportPath}`);
+    return summary.failed > 0 ? 1 : 0;
+}

package/esm/src/eval/agent-service/live-evals/environment.d.ts

@@ -0,0 +1,15 @@
+import { type AgentServiceConfigInput } from "../../../agent/service/config.js";
+/** Public API contract for live eval environment. */
+export interface LiveEvalEnvironment {
+    endpoint: string;
+    authToken: string;
+    apiUrl: string;
+    projectId: string | undefined;
+    branchId: string | undefined;
+    model: string | undefined;
+}
+/** Default value for live eval endpoint. */
+export declare const DEFAULT_LIVE_EVAL_ENDPOINT = "http://127.0.0.1:3001/api/ag-ui";
+/** Resolves live eval environment. */
+export declare function resolveLiveEvalEnvironment(env?: AgentServiceConfigInput): LiveEvalEnvironment;
+//# sourceMappingURL=environment.d.ts.map

package/esm/src/eval/agent-service/live-evals/environment.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"environment.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/live-evals/environment.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,uBAAuB,EAE7B,MAAM,kCAAkC,CAAC;AAE1C,qDAAqD;AACrD,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3B;AAED,4CAA4C;AAC5C,eAAO,MAAM,0BAA0B,oCAAoC,CAAC;AAE5E,sCAAsC;AACtC,wBAAgB,0BAA0B,CACxC,GAAG,GAAE,uBAA4B,GAChC,mBAAmB,CAerB"}

package/esm/src/eval/agent-service/live-evals/environment.js

@@ -0,0 +1,20 @@
+import { parseAgentServiceConfig, } from "../../../agent/service/config.js";
+/** Default value for live eval endpoint. */
+export const DEFAULT_LIVE_EVAL_ENDPOINT = "http://127.0.0.1:3001/api/ag-ui";
+/** Resolves live eval environment. */
+export function resolveLiveEvalEnvironment(env = {}) {
+    return {
+        endpoint: typeof env.AG_UI_EVAL_ENDPOINT === "string"
+            ? env.AG_UI_EVAL_ENDPOINT
+            : DEFAULT_LIVE_EVAL_ENDPOINT,
+        authToken: typeof env.VERYFRONT_TOKEN === "string" ? env.VERYFRONT_TOKEN : "",
+        apiUrl: typeof env.VERYFRONT_API_URL === "string"
+            ? env.VERYFRONT_API_URL
+            : parseAgentServiceConfig(env).VERYFRONT_API_URL,
+        projectId: typeof env.AG_UI_EVAL_PROJECT_ID === "string"
+            ? env.AG_UI_EVAL_PROJECT_ID
+            : undefined,
+        branchId: typeof env.AG_UI_EVAL_BRANCH_ID === "string" ? env.AG_UI_EVAL_BRANCH_ID : undefined,
+        model: typeof env.AG_UI_EVAL_MODEL === "string" ? env.AG_UI_EVAL_MODEL : undefined,
+    };
+}

package/esm/src/eval/agent-service/live-evals/formatting.d.ts

@@ -0,0 +1,15 @@
+import { Buffer } from "node:buffer";
+import type { AgUiSseProgressSnapshot as EvalProgressSnapshot } from "../../../agent/index.js";
+/** Create plain text pdf. */
+export declare function createPlainTextPdf(lines: string[]): Buffer;
+/** Builds progress line. */
+export declare function buildProgressLine(input: {
+    caseId: string;
+    startedAt: number;
+    progress: EvalProgressSnapshot;
+}): string;
+/** Builds failure suffix. */
+export declare function buildFailureSuffix(progress: EvalProgressSnapshot): string;
+/** Contains ordered subsequence helper. */
+export declare function containsOrderedSubsequence(haystack: string[], needle: string[]): boolean;
+//# sourceMappingURL=formatting.d.ts.map

package/esm/src/eval/agent-service/live-evals/formatting.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"formatting.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/live-evals/formatting.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,KAAK,EAAE,uBAAuB,IAAI,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAS/F,6BAA6B;AAC7B,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CA4C1D;AAED,4BAA4B;AAC5B,wBAAgB,iBAAiB,CAAC,KAAK,EAAE;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,oBAAoB,CAAC;CAChC,GAAG,MAAM,CAcT;AAED,6BAA6B;AAC7B,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,oBAAoB,GAAG,MAAM,CAUzE;AAED,2CAA2C;AAC3C,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,MAAM,EAAE,EAClB,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAcT"}

package/esm/src/eval/agent-service/live-evals/formatting.js

@@ -0,0 +1,75 @@
+import { Buffer } from "node:buffer";
+function escapePdfText(input) {
+    return input.replaceAll("\\", "\\\\").replaceAll("(", "\\(").replaceAll(")", "\\)");
+}
+/** Create plain text pdf. */
+export function createPlainTextPdf(lines) {
+    const contentLines = [
+        "BT",
+        "/F1 18 Tf",
+        "72 720 Td",
+        `(${escapePdfText(lines[0] ?? "Untitled")}) Tj`,
+        "/F1 12 Tf",
+        ...lines.slice(1).flatMap((line) => ["0 -20 Td", `(${escapePdfText(line)}) Tj`]),
+        "ET",
+    ];
+    const contentStream = contentLines.join("\n");
+    const objects = [
+        "<< /Type /Catalog /Pages 2 0 R >>",
+        "<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+        "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>",
+        `<< /Length ${Buffer.byteLength(contentStream, "utf8")} >>\nstream\n${contentStream}\nendstream`,
+        "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+    ];
+    let pdf = "%PDF-1.4\n";
+    const offsets = [0];
+    objects.forEach((objectBody, index) => {
+        offsets.push(Buffer.byteLength(pdf, "utf8"));
+        pdf += `${index + 1} 0 obj\n${objectBody}\nendobj\n`;
+    });
+    const xrefOffset = Buffer.byteLength(pdf, "utf8");
+    pdf += `xref\n0 ${objects.length + 1}\n`;
+    pdf += "0000000000 65535 f \n";
+    offsets.slice(1).forEach((offset) => {
+        pdf += `${offset.toString().padStart(10, "0")} 00000 n \n`;
+    });
+    pdf += `trailer\n<< /Root 1 0 R /Size ${objects.length + 1} >>\nstartxref\n${xrefOffset}\n%%EOF\n`;
+    return Buffer.from(pdf, "utf8");
+}
+/** Builds progress line. */
+export function buildProgressLine(input) {
+    const elapsedSeconds = Math.max(1, Math.round((Date.now() - input.startedAt) / 1000));
+    const lastEvent = input.progress.lastEventType ?? "none";
+    const lastTool = input.progress.lastToolCallName
+        ? ` tool=${input.progress.lastToolCallName}`
+        : "";
+    const toolSummary = input.progress.toolStarts.length > 0
+        ? ` tools=${input.progress.toolStarts.join(",")}`
+        : "";
+    const textSummary = input.progress.textLength > 0 ? ` text=${input.progress.textLength}ch` : "";
+    return `[progress] ${input.caseId} ${elapsedSeconds}s events=${input.progress.eventCount} last=${lastEvent}${lastTool}${toolSummary}${textSummary}`;
+}
+/** Builds failure suffix. */
+export function buildFailureSuffix(progress) {
+    const details = [
+        `events=${progress.eventCount}`,
+        `last=${progress.lastEventType ?? "none"}`,
+        progress.lastToolCallName ? `tool=${progress.lastToolCallName}` : null,
+        progress.toolStarts.length > 0 ? `tools=${progress.toolStarts.join(",")}` : null,
+        progress.textLength > 0 ? `text=${progress.textLength}ch` : null,
+    ].filter((value) => value !== null);
+    return details.length > 0 ? ` Progress: ${details.join(" ")}` : "";
+}
+/** Contains ordered subsequence helper. */
+export function containsOrderedSubsequence(haystack, needle) {
+    let cursor = 0;
+    for (const value of haystack) {
+        if (value === needle[cursor]) {
+            cursor += 1;
+        }
+        if (cursor === needle.length) {
+            return true;
+        }
+    }
+    return cursor === needle.length;
+}

package/esm/src/eval/agent-service/live-evals/index.d.ts

@@ -0,0 +1,12 @@
+export { type LiveEvalCliCaseFactoryInput, type LiveEvalCliCaseGroups, runLiveEvalCli, type RunLiveEvalCliInput, } from "./cli-runner.js";
+export { DEFAULT_LIVE_EVAL_ENDPOINT, type LiveEvalEnvironment, resolveLiveEvalEnvironment, } from "./environment.js";
+export { cancelLiveEvalInputRequest, createLiveEvalApiClient, createLiveEvalConversation, createLiveEvalProjectUploadFixture, createLiveEvalRelease, deleteLiveEvalConversation, deleteLiveEvalProjectFile, getLiveEvalProjectFile, listOpenLiveEvalInputRequests, type LiveEvalApiClient, type LiveEvalApiContext, type LiveEvalConversationInput, type LiveEvalCreateConversationInput, type LiveEvalCreateReleaseInput, type LiveEvalInputRequestInput, type LiveEvalInputRequestRecord, type LiveEvalInputResponseValues, type LiveEvalProjectFileInput, type LiveEvalProjectUploadFixtureInput, type LiveEvalRequestTimeoutInput, type LiveEvalSubmitInputResponseInput, type LiveEvalWaitForOpenInputRequestInput, submitLiveEvalInputResponse, waitForOpenLiveEvalInputRequest, } from "./api-client.js";
+export { buildFailureSuffix, buildProgressLine, containsOrderedSubsequence, createPlainTextPdf, } from "./formatting.js";
+export { evaluateRuntimeConfidenceEnv, printRuntimeConfidencePreflight, type RuntimeConfidencePreflightResult, } from "./preflight.js";
+export { buildRuntimePerformanceSummary, type LiveEvalResultForPerformance, type LiveEvalRuntime, type RuntimePerformanceSummary, } from "./performance.js";
+export { buildLiveEvalCaseMetadata, type BuildLiveEvalCaseMetadataInput, DEFAULT_LIVE_EVAL_AREA_TAG_RULES, DEFAULT_LIVE_EVAL_OPTIONAL_JUDGE_CASE_PREFIXES, type LiveEvalCaseMetadataOptions, type LiveEvalCaseSurface, type LiveEvalCaseTagRule, withLiveEvalMetadata, } from "./metadata.js";
+export { buildLiveEvalRequestBody, type BuildLiveEvalRequestBodyInput, type LiveEvalRequestBody, } from "./request.js";
+export { buildLiveEvalCaseTagSummary, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, hasEveryLiveEvalTag, type LiveEvalCaseMetadata, type LiveEvalCaseSelectionInput, type LiveEvalResultForReport, resolveLiveEvalRequestedCaseIds, selectLiveEvalCases, } from "./report.js";
+export { createFailedEvalResult, createPassedEvalResult, createSkippedEvalResult, type LiveEvalResultRecord, } from "./result.js";
+export { containsSkillLoad, countStepStartedEvents, createLiveEvalCaseSupport, hasFinished, type LiveEvalCase, type LiveEvalContext, type LiveEvalProjectFile, type LiveEvalProjectFileReaderInput, type LiveEvalRunnerConfig, liveEvalRunnerInternals, type PreparedLiveEvalInput, } from "./runner.js";
+//# sourceMappingURL=index.d.ts.map

package/esm/src/eval/agent-service/live-evals/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/live-evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,2BAA2B,EAChC,KAAK,qBAAqB,EAC1B,cAAc,EACd,KAAK,mBAAmB,GACzB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,0BAA0B,EAC1B,KAAK,mBAAmB,EACxB,0BAA0B,GAC3B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,0BAA0B,EAC1B,uBAAuB,EACvB,0BAA0B,EAC1B,kCAAkC,EAClC,qBAAqB,EACrB,0BAA0B,EAC1B,yBAAyB,EACzB,sBAAsB,EACtB,6BAA6B,EAC7B,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,yBAAyB,EAC9B,KAAK,+BAA+B,EACpC,KAAK,0BAA0B,EAC/B,KAAK,yBAAyB,EAC9B,KAAK,0BAA0B,EAC/B,KAAK,2BAA2B,EAChC,KAAK,wBAAwB,EAC7B,KAAK,iCAAiC,EACtC,KAAK,2BAA2B,EAChC,KAAK,gCAAgC,EACrC,KAAK,oCAAoC,EACzC,2BAA2B,EAC3B,+BAA+B,GAChC,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,0BAA0B,EAC1B,kBAAkB,GACnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,4BAA4B,EAC5B,+BAA+B,EAC/B,KAAK,gCAAgC,GACtC,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,8BAA8B,EAC9B,KAAK,4BAA4B,EACjC,KAAK,eAAe,EACpB,KAAK,yBAAyB,GAC/B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,yBAAyB,EACzB,KAAK,8BAA8B,EACnC,gCAAgC,EAChC,8CAA8C,EAC9C,KAAK,2BAA2B,EAChC,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,oBAAoB,GACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,wBAAwB,EACxB,KAAK,6BAA6B,EAClC,KAAK,mBAAmB,GACzB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,2BAA2B,EAC3B,2BAA2B,EAC3B,0BAA0B,EAC1B,mBAAmB,EACnB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,EAC/B,KAAK,uBAAuB,EAC5B,+BAA+B,EAC/B,mBAAmB,GACpB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,KAAK,oBAAoB,GAC1B,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,iBAAiB,EACjB,sBAAsB,EACtB,yBAAyB,EACzB,WAAW,EACX,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,8BAA8B,EACnC,KAAK,oBAAoB,EACzB,uBAAuB,EACvB,KAAK,qBAAqB,GAC3B,MAAM,aAAa,CAAC"}

package/esm/src/eval/agent-service/live-evals/index.js

@@ -0,0 +1,11 @@
+export { runLiveEvalCli, } from "./cli-runner.js";
+export { DEFAULT_LIVE_EVAL_ENDPOINT, resolveLiveEvalEnvironment, } from "./environment.js";
+export { cancelLiveEvalInputRequest, createLiveEvalApiClient, createLiveEvalConversation, createLiveEvalProjectUploadFixture, createLiveEvalRelease, deleteLiveEvalConversation, deleteLiveEvalProjectFile, getLiveEvalProjectFile, listOpenLiveEvalInputRequests, submitLiveEvalInputResponse, waitForOpenLiveEvalInputRequest, } from "./api-client.js";
+export { buildFailureSuffix, buildProgressLine, containsOrderedSubsequence, createPlainTextPdf, } from "./formatting.js";
+export { evaluateRuntimeConfidenceEnv, printRuntimeConfidencePreflight, } from "./preflight.js";
+export { buildRuntimePerformanceSummary, } from "./performance.js";
+export { buildLiveEvalCaseMetadata, DEFAULT_LIVE_EVAL_AREA_TAG_RULES, DEFAULT_LIVE_EVAL_OPTIONAL_JUDGE_CASE_PREFIXES, withLiveEvalMetadata, } from "./metadata.js";
+export { buildLiveEvalRequestBody, } from "./request.js";
+export { buildLiveEvalCaseTagSummary, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, hasEveryLiveEvalTag, resolveLiveEvalRequestedCaseIds, selectLiveEvalCases, } from "./report.js";
+export { createFailedEvalResult, createPassedEvalResult, createSkippedEvalResult, } from "./result.js";
+export { containsSkillLoad, countStepStartedEvents, createLiveEvalCaseSupport, hasFinished, liveEvalRunnerInternals, } from "./runner.js";

package/esm/src/eval/agent-service/live-evals/metadata.d.ts

@@ -0,0 +1,34 @@
+import type { LiveEvalCaseMetadata } from "./report.js";
+import type { LiveEvalCase } from "./runner.js";
+/** Public API contract for live eval case surface. */
+export type LiveEvalCaseSurface = "read-only" | "write" | "experimental";
+/** Public API contract for live eval case tag rule. */
+export interface LiveEvalCaseTagRule {
+    tag: string;
+    equals?: string | readonly string[];
+    startsWith?: string | readonly string[];
+    includes?: string | readonly string[];
+}
+/** Options accepted by live eval case metadata. */
+export interface LiveEvalCaseMetadataOptions {
+    releaseGateCaseIds?: readonly string[] | ReadonlySet<string>;
+    optionalJudgeCasePrefixes?: readonly string[];
+    areaTagRules?: readonly LiveEvalCaseTagRule[];
+}
+/** Input payload for build live eval case metadata. */
+export interface BuildLiveEvalCaseMetadataInput extends LiveEvalCaseMetadataOptions {
+    caseId: string;
+    surface: LiveEvalCaseSurface;
+    requireProject: boolean;
+}
+/** Default value for live eval optional judge case prefixes. */
+export declare const DEFAULT_LIVE_EVAL_OPTIONAL_JUDGE_CASE_PREFIXES: readonly string[];
+/** Default value for live eval area tag rules. */
+export declare const DEFAULT_LIVE_EVAL_AREA_TAG_RULES: readonly LiveEvalCaseTagRule[];
+/** Builds live eval case metadata. */
+export declare function buildLiveEvalCaseMetadata(input: BuildLiveEvalCaseMetadataInput): LiveEvalCaseMetadata;
+/** Applies live eval metadata. */
+export declare function withLiveEvalMetadata<TCase extends LiveEvalCase>(cases: readonly TCase[], surface: LiveEvalCaseSurface, options?: LiveEvalCaseMetadataOptions): Array<TCase & {
+    metadata: LiveEvalCaseMetadata;
+}>;
+//# sourceMappingURL=metadata.d.ts.map

package/esm/src/eval/agent-service/live-evals/metadata.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metadata.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/live-evals/metadata.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,sDAAsD;AACtD,MAAM,MAAM,mBAAmB,GAAG,WAAW,GAAG,OAAO,GAAG,cAAc,CAAC;AAEzE,uDAAuD;AACvD,MAAM,WAAW,mBAAmB;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAAC;IACpC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAAC;CACvC;AAED,mDAAmD;AACnD,MAAM,WAAW,2BAA2B;IAC1C,kBAAkB,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IAC7D,yBAAyB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9C,YAAY,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAC;CAC/C;AAED,uDAAuD;AACvD,MAAM,WAAW,8BAA+B,SAAQ,2BAA2B;IACjF,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,mBAAmB,CAAC;IAC7B,cAAc,EAAE,OAAO,CAAC;CACzB;AAED,gEAAgE;AAChE,eAAO,MAAM,8CAA8C,EAAE,SAAS,MAAM,EAI3E,CAAC;AAEF,kDAAkD;AAClD,eAAO,MAAM,gCAAgC,EAAE,SAAS,mBAAmB,EAqB1E,CAAC;AA+DF,sCAAsC;AACtC,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,8BAA8B,GACpC,oBAAoB,CAkCtB;AAED,kCAAkC;AAClC,wBAAgB,oBAAoB,CAAC,KAAK,SAAS,YAAY,EAC7D,KAAK,EAAE,SAAS,KAAK,EAAE,EACvB,OAAO,EAAE,mBAAmB,EAC5B,OAAO,GAAE,2BAAgC,GACxC,KAAK,CAAC,KAAK,GAAG;IAAE,QAAQ,EAAE,oBAAoB,CAAA;CAAE,CAAC,CAUnD"}

package/esm/src/eval/agent-service/live-evals/metadata.js

@@ -0,0 +1,116 @@
+/** Default value for live eval optional judge case prefixes. */
+export const DEFAULT_LIVE_EVAL_OPTIONAL_JUDGE_CASE_PREFIXES = [
+    "knowledge-",
+    "grounded-",
+    "judged-",
+];
+/** Default value for live eval area tag rules. */
+export const DEFAULT_LIVE_EVAL_AREA_TAG_RULES = [
+    { startsWith: "starter-", tag: "area:starter-routing" },
+    { startsWith: "starter-task-", tag: "area:starter-artifact-flow" },
+    { startsWith: "starter-", tag: "behavior:conversation-first" },
+    { startsWith: "workflow-", tag: "area:workflow" },
+    { startsWith: "platform-", tag: "area:platform" },
+    { startsWith: "security-", tag: "area:security" },
+    { startsWith: ["knowledge-", "grounded-", "judged-"], tag: "area:knowledge" },
+    { startsWith: "tool-truthfulness", tag: "area:tool-truthfulness" },
+    { startsWith: "degraded-", tag: "area:resilience" },
+    { equals: "error-recovery-missing-file", tag: "area:resilience" },
+    { includes: "deploy", tag: "area:deployment" },
+    { includes: "sandbox", tag: "area:sandbox" },
+    { includes: "debug", tag: "area:debugging" },
+    { includes: "operate", tag: "area:operations" },
+    { includes: "research", tag: "area:research" },
+    { includes: "knowledge", tag: "area:knowledge-lifecycle" },
+    { includes: "agent", tag: "area:agent-authoring" },
+    { includes: "form-input", tag: "area:interactive-input" },
+    { includes: ["invoke-agent", "delegation"], tag: "area:delegation" },
+    { includes: ["create-page", "create-api-route", "create-skill"], tag: "area:file-generation" },
+];
+function toStringArray(value) {
+    if (!value) {
+        return [];
+    }
+    return typeof value === "string" ? [value] : value;
+}
+function caseIdCollectionHas(collection, caseId) {
+    if (!collection) {
+        return false;
+    }
+    if ("has" in collection) {
+        return collection.has(caseId);
+    }
+    return collection.includes(caseId);
+}
+function matchesTagRule(caseId, rule) {
+    for (const value of toStringArray(rule.equals)) {
+        if (caseId === value) {
+            return true;
+        }
+    }
+    for (const value of toStringArray(rule.startsWith)) {
+        if (caseId.startsWith(value)) {
+            return true;
+        }
+    }
+    for (const value of toStringArray(rule.includes)) {
+        if (caseId.includes(value)) {
+            return true;
+        }
+    }
+    return false;
+}
+function isOptionalJudgeCase(caseId, prefixes) {
+    return prefixes.some((prefix) => caseId.startsWith(prefix));
+}
+function buildAreaTags(caseId, rules) {
+    const tags = new Set();
+    for (const rule of rules) {
+        if (matchesTagRule(caseId, rule)) {
+            tags.add(rule.tag);
+        }
+    }
+    return [...tags];
+}
+/** Builds live eval case metadata. */
+export function buildLiveEvalCaseMetadata(input) {
+    const optionalJudgeCasePrefixes = input.optionalJudgeCasePrefixes ??
+        DEFAULT_LIVE_EVAL_OPTIONAL_JUDGE_CASE_PREFIXES;
+    const areaTagRules = input.areaTagRules ?? DEFAULT_LIVE_EVAL_AREA_TAG_RULES;
+    const gradingTag = isOptionalJudgeCase(input.caseId, optionalJudgeCasePrefixes)
+        ? "grading:deterministic-plus-optional-llm"
+        : "grading:deterministic-only";
+    const tags = new Set([
+        `surface:${input.surface}`,
+        input.requireProject ? "project:required" : "project:optional",
+        input.surface === "experimental" ? "stability:experimental" : "stability:stable",
+        gradingTag,
+    ]);
+    if (input.surface !== "experimental") {
+        tags.add("gate:nightly");
+    }
+    if (gradingTag === "grading:deterministic-only" && input.surface !== "experimental") {
+        tags.add("gate:ci");
+    }
+    if (caseIdCollectionHas(input.releaseGateCaseIds, input.caseId)) {
+        tags.add("gate:release");
+    }
+    for (const tag of buildAreaTags(input.caseId, areaTagRules)) {
+        tags.add(tag);
+    }
+    return {
+        tags: [...tags].sort(),
+    };
+}
+/** Applies live eval metadata. */
+export function withLiveEvalMetadata(cases, surface, options = {}) {
+    return cases.map((testCase) => ({
+        ...testCase,
+        metadata: buildLiveEvalCaseMetadata({
+            ...options,
+            caseId: testCase.id,
+            surface,
+            requireProject: testCase.requireProject === true,
+        }),
+    }));
+}

package/esm/src/eval/agent-service/live-evals/performance.d.ts

@@ -0,0 +1,19 @@
+/** Public API contract for live eval runtime. */
+export type LiveEvalRuntime = "framework";
+/** Public API contract for live eval result for performance. */
+export interface LiveEvalResultForPerformance {
+    runtime: LiveEvalRuntime;
+    durationMs: number;
+}
+/** Public API contract for runtime performance summary. */
+export interface RuntimePerformanceSummary {
+    count: number;
+    avgDurationMs: number;
+    p50DurationMs: number;
+    p95DurationMs: number;
+    minDurationMs: number;
+    maxDurationMs: number;
+}
+/** Builds runtime performance summary. */
+export declare function buildRuntimePerformanceSummary(results: LiveEvalResultForPerformance[]): Record<LiveEvalRuntime, RuntimePerformanceSummary>;
+//# sourceMappingURL=performance.d.ts.map

package/esm/src/eval/agent-service/live-evals/performance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"performance.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/live-evals/performance.ts"],"names":[],"mappings":"AAAA,iDAAiD;AACjD,MAAM,MAAM,eAAe,GAAG,WAAW,CAAC;AAE1C,gEAAgE;AAChE,MAAM,WAAW,4BAA4B;IAC3C,OAAO,EAAE,eAAe,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,2DAA2D;AAC3D,MAAM,WAAW,yBAAyB;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB;AAkBD,0CAA0C;AAC1C,wBAAgB,8BAA8B,CAC5C,OAAO,EAAE,4BAA4B,EAAE,GACtC,MAAM,CAAC,eAAe,EAAE,yBAAyB,CAAC,CAiBpD"}

package/esm/src/eval/agent-service/live-evals/performance.js

@@ -0,0 +1,22 @@
+function calculateDurationPercentile(sortedDurations, percentile) {
+    if (sortedDurations.length === 0) {
+        return 0;
+    }
+    const index = Math.min(sortedDurations.length - 1, Math.max(0, Math.ceil((percentile / 100) * sortedDurations.length) - 1));
+    return sortedDurations[index] ?? 0;
+}
+/** Builds runtime performance summary. */
+export function buildRuntimePerformanceSummary(results) {
+    const durations = results.map((result) => result.durationMs).sort((left, right) => left - right);
+    const totalDuration = durations.reduce((sum, value) => sum + value, 0);
+    return {
+        framework: {
+            count: durations.length,
+            avgDurationMs: durations.length > 0 ? Math.round(totalDuration / durations.length) : 0,
+            p50DurationMs: calculateDurationPercentile(durations, 50),
+            p95DurationMs: calculateDurationPercentile(durations, 95),
+            minDurationMs: durations[0] ?? 0,
+            maxDurationMs: durations.at(-1) ?? 0,
+        },
+    };
+}

package/esm/src/eval/agent-service/live-evals/preflight.d.ts

@@ -0,0 +1,12 @@
+import { type AgentServiceConfigInput } from "../../../agent/service/config.js";
+/** Result returned from runtime confidence preflight. */
+export interface RuntimeConfidencePreflightResult {
+    ok: boolean;
+    resolvedApiUrl: string;
+    messages: string[];
+}
+/** Evaluate runtime confidence env helper. */
+export declare function evaluateRuntimeConfidenceEnv(env?: AgentServiceConfigInput, resolvedApiUrl?: string): RuntimeConfidencePreflightResult;
+/** Print runtime confidence preflight helper. */
+export declare function printRuntimeConfidencePreflight(result: RuntimeConfidencePreflightResult, output?: Pick<Console, "error" | "log">): void;
+//# sourceMappingURL=preflight.d.ts.map

package/esm/src/eval/agent-service/live-evals/preflight.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"preflight.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/live-evals/preflight.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,uBAAuB,EAE7B,MAAM,kCAAkC,CAAC;AAE1C,yDAAyD;AACzD,MAAM,WAAW,gCAAgC;IAC/C,EAAE,EAAE,OAAO,CAAC;IACZ,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,8CAA8C;AAC9C,wBAAgB,4BAA4B,CAC1C,GAAG,GAAE,uBAA4B,EACjC,cAAc,GAAE,MAAuD,GACtE,gCAAgC,CAoBlC;AAED,iDAAiD;AACjD,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,gCAAgC,EACxC,MAAM,GAAE,IAAI,CAAC,OAAO,EAAE,OAAO,GAAG,KAAK,CAAW,GAC/C,IAAI,CAKN"}

package/esm/src/eval/agent-service/live-evals/preflight.js

@@ -0,0 +1,27 @@
+import { parseAgentServiceConfig, } from "../../../agent/service/config.js";
+/** Evaluate runtime confidence env helper. */
+export function evaluateRuntimeConfidenceEnv(env = {}, resolvedApiUrl = parseAgentServiceConfig(env).VERYFRONT_API_URL) {
+    const messages = [`Resolved VERYFRONT_API_URL: ${resolvedApiUrl}`];
+    let hasBlockers = false;
+    if (typeof env.VERYFRONT_TOKEN !== "string" || env.VERYFRONT_TOKEN.length === 0) {
+        hasBlockers = true;
+        messages.push("BLOCKER: VERYFRONT_TOKEN is missing");
+    }
+    if (typeof env.AG_UI_EVAL_PROJECT_ID !== "string" || env.AG_UI_EVAL_PROJECT_ID.length === 0) {
+        hasBlockers = true;
+        messages.push("BLOCKER: AG_UI_EVAL_PROJECT_ID is missing");
+    }
+    if (!hasBlockers) {
+        messages.push("Runtime-confidence preflight: PASS");
+        return { ok: true, resolvedApiUrl, messages };
+    }
+    messages.push("Runtime-confidence preflight: FAIL");
+    return { ok: false, resolvedApiUrl, messages };
+}
+/** Print runtime confidence preflight helper. */
+export function printRuntimeConfidencePreflight(result, output = console) {
+    for (const message of result.messages) {
+        const writer = result.ok ? output.log : output.error;
+        writer(message);
+    }
+}

package/esm/src/eval/agent-service/live-evals/report.d.ts

@@ -0,0 +1,55 @@
+import type { LiveEvalRuntime } from "./performance.js";
+/** Public API contract for live eval case metadata. */
+export interface LiveEvalCaseMetadata {
+    tags: readonly string[];
+}
+/** Public API contract for live eval result for report. */
+export interface LiveEvalResultForReport {
+    runtime: LiveEvalRuntime;
+    status: "pass" | "fail" | "skip";
+}
+/** Input payload for live eval case selection. */
+export interface LiveEvalCaseSelectionInput<TCase extends {
+    id: string;
+}> {
+    allCases: TCase[];
+    readOnlyCases: TCase[];
+    writeCases: TCase[];
+    experimentalWriteCases: TCase[];
+    requestedCaseIds: Set<string>;
+    requestedCaseTags?: Set<string>;
+    runWriteEvals: boolean;
+    runExperimentalWriteEvals: boolean;
+}
+/** Check whether every live eval tag is present. */
+export declare function hasEveryLiveEvalTag(tags: readonly string[], requestedTags: Set<string>): boolean;
+/** Builds live eval case tag summary. */
+export declare function buildLiveEvalCaseTagSummary(cases: readonly {
+    metadata?: LiveEvalCaseMetadata;
+}[]): Record<string, number>;
+/** Select live eval cases helper. */
+export declare function selectLiveEvalCases<TCase extends {
+    id: string;
+    metadata?: LiveEvalCaseMetadata;
+}>(input: LiveEvalCaseSelectionInput<TCase>): TCase[];
+/** Resolves live eval requested case IDs. */
+export declare function resolveLiveEvalRequestedCaseIds(input: {
+    caseSets: Record<string, readonly string[]>;
+    requestedCaseIds: Set<string>;
+    requestedCaseSetId?: string | null;
+}): Set<string>;
+interface LiveEvalRuntimeCounts {
+    passed: number;
+    failed: number;
+    skipped: number;
+}
+/** Builds live eval runtime summary. */
+export declare function buildLiveEvalRuntimeSummary(runtimes: readonly LiveEvalRuntime[], results: LiveEvalResultForReport[]): Record<LiveEvalRuntime, LiveEvalRuntimeCounts>;
+/** Builds live eval status summary. */
+export declare function buildLiveEvalStatusSummary(results: LiveEvalResultForReport[]): {
+    passed: number;
+    failed: number;
+    skipped: number;
+};
+export {};
+//# sourceMappingURL=report.d.ts.map

package/esm/src/eval/agent-service/live-evals/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/live-evals/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAExD,uDAAuD;AACvD,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;CACzB;AAED,2DAA2D;AAC3D,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,eAAe,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;CAClC;AAED,kDAAkD;AAClD,MAAM,WAAW,0BAA0B,CAAC,KAAK,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE;IACtE,QAAQ,EAAE,KAAK,EAAE,CAAC;IAClB,aAAa,EAAE,KAAK,EAAE,CAAC;IACvB,UAAU,EAAE,KAAK,EAAE,CAAC;IACpB,sBAAsB,EAAE,KAAK,EAAE,CAAC;IAChC,gBAAgB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9B,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAChC,aAAa,EAAE,OAAO,CAAC;IACvB,yBAAyB,EAAE,OAAO,CAAC;CACpC;AAED,oDAAoD;AACpD,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,GACzB,OAAO,CAQT;AAED,yCAAyC;AACzC,wBAAgB,2BAA2B,CACzC,KAAK,EAAE,SAAS;IACd,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,EAAE,GACF,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAaxB;AAED,qCAAqC;AACrC,wBAAgB,mBAAmB,CACjC,KAAK,SAAS;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,oBAAoB,CAAA;CAAE,EAE7D,KAAK,EAAE,0BAA0B,CAAC,KAAK,CAAC,GACvC,KAAK,EAAE,CAoBT;AAED,6CAA6C;AAC7C,wBAAgB,+BAA+B,CAAC,KAAK,EAAE;IACrD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;IAC5C,gBAAgB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,GAAG,GAAG,CAAC,MAAM,CAAC,CAsBd;AAED,UAAU,qBAAqB;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAgBD,wCAAwC;AACxC,wBAAgB,2BAA2B,CACzC,QAAQ,EAAE,SAAS,eAAe,EAAE,EACpC,OAAO,EAAE,uBAAuB,EAAE,GACjC,MAAM,CAAC,eAAe,EAAE,qBAAqB,CAAC,CAShD;AAED,uCAAuC;AACvC,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,uBAAuB,EAAE,GACjC;IACD,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB,CAMA"}