veryfront 0.1.522 → 0.1.523

package/esm/src/agent/testing/live-evals/runner.js

@@ -0,0 +1,391 @@
+import * as dntShim from "../../../../_dnt.shims.js";
+import { agUiSseEventTypes, buildAgUiSseTraceSignature as buildTraceSignature, getAgUiSseStringField as getStringField, parseAgUiSseResponse as parseSseResponse, } from "../../index.js";
+import { buildFailureSuffix, buildProgressLine, containsOrderedSubsequence } from "./formatting.js";
+import { buildLiveEvalRequestBody } from "./request.js";
+import { createFailedEvalResult, createPassedEvalResult, createSkippedEvalResult, } from "./result.js";
+function resolveFetch(config) {
+    return config.fetch ?? fetch;
+}
+function createLiveEvalJudgeSupport(config) {
+    async function judgeLlm(input) {
+        try {
+            const body = buildLiveEvalRequestBody({
+                testCaseId: "llm-judge",
+                prompt: `You are an eval judge. Grade the following answer.
+
+QUESTION: ${input.question}
+
+ANSWER: ${input.answer}
+
+CRITERIA: ${input.criteria}
+
+Respond with exactly one line: PASS or FAIL followed by a brief reason.
+Example: "PASS — correctly explains the pattern with accurate details"
+Example: "FAIL — mentions the wrong file convention"`,
+                projectId: null,
+                allowedTools: [],
+                forceRuntimeOverrides: true,
+                maxSteps: 2,
+            });
+            const response = await resolveFetch(config)(config.endpoint, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${config.authToken}`,
+                },
+                body: JSON.stringify(body),
+                signal: AbortSignal.timeout(30_000),
+            });
+            const run = await parseSseResponse(response);
+            if (run.responseStatus !== 200) {
+                return { pass: false, reason: `judge returned HTTP ${run.responseStatus}` };
+            }
+            const line = run.text
+                .split("\n")
+                .map((value) => value.trim())
+                .find((value) => value.length > 0) ?? "";
+            if (line.toUpperCase().startsWith("PASS")) {
+                return { pass: true, reason: line };
+            }
+            return { pass: false, reason: line || "judge returned no decision" };
+        }
+        catch (error) {
+            return {
+                pass: false,
+                reason: error instanceof Error ? error.message : String(error),
+            };
+        }
+    }
+    function withJudge(structuralVerify, judgeInput) {
+        return async (run) => {
+            const structuralFailure = structuralVerify(run);
+            if (structuralFailure) {
+                return structuralFailure;
+            }
+            if (!config.enableLlmJudge) {
+                return null;
+            }
+            const judgment = await judgeLlm({
+                question: judgeInput.question,
+                answer: run.text,
+                criteria: judgeInput.criteria,
+            });
+            return judgment.pass ? null : `LLM judge: ${judgment.reason}`;
+        };
+    }
+    return {
+        judgeLlm,
+        withJudge,
+    };
+}
+function createInitialProgressSnapshot() {
+    return {
+        eventCount: 0,
+        lastEventType: null,
+        lastToolCallName: null,
+        toolStarts: [],
+        textLength: 0,
+    };
+}
+function isUnrefableTimer(value) {
+    return typeof value === "object" && value !== null && "unref" in value &&
+        typeof value.unref === "function";
+}
+function maybeUnrefTimer(timer) {
+    if (isUnrefableTimer(timer)) {
+        timer.unref();
+    }
+}
+function createLiveEvalProgressReporter(input) {
+    let latestProgress = createInitialProgressSnapshot();
+    const progressTimer = dntShim.setInterval(() => {
+        input.log(buildProgressLine({
+            caseId: input.caseId,
+            startedAt: input.startedAt,
+            progress: latestProgress,
+        }));
+    }, input.intervalMs);
+    maybeUnrefTimer(progressTimer);
+    return {
+        stop: () => {
+            clearInterval(progressTimer);
+        },
+        update: (snapshot) => {
+            latestProgress = snapshot;
+        },
+        getSnapshot: () => latestProgress,
+    };
+}
+function collectPreparedArtifactPaths(prepared) {
+    if (!prepared?.metadata) {
+        return [];
+    }
+    return [
+        ...new Set(Object.entries(prepared.metadata)
+            .filter(([key, value]) => key.toLowerCase().includes("path") && value.length > 0)
+            .map(([, value]) => value)),
+    ].sort();
+}
+function extractPreparedConversationId(prepared) {
+    return typeof prepared?.metadata?.conversationId === "string" &&
+        prepared.metadata.conversationId.length > 0
+        ? prepared.metadata.conversationId
+        : null;
+}
+function createLiveEvalRunArtifacts(input) {
+    return {
+        ...(input.runId ? { runId: input.runId } : {}),
+        traceSignature: input.traceSignature,
+        toolStarts: input.run.toolStarts,
+        toolArgsPreview: input.run.toolArgs.join(" | ").slice(0, 1000),
+        textPreview: input.run.text.slice(0, 280),
+    };
+}
+function createFailedRunEvalResult(input) {
+    return createFailedEvalResult({
+        id: input.context.id,
+        label: input.context.label,
+        runtime: input.context.runtime,
+        details: input.details,
+        startedAt: input.context.startedAt,
+        ...(input.context.conversationId ? { conversationId: input.context.conversationId } : {}),
+        ...(input.runArtifacts.runId ? { runId: input.runArtifacts.runId } : {}),
+        ...(input.context.artifactPaths?.length ? { artifactPaths: input.context.artifactPaths } : {}),
+        traceSignature: input.runArtifacts.traceSignature,
+        toolStarts: input.runArtifacts.toolStarts,
+        toolArgsPreview: input.runArtifacts.toolArgsPreview,
+        textPreview: input.runArtifacts.textPreview,
+    });
+}
+function createPassedRunEvalResult(input) {
+    return createPassedEvalResult({
+        id: input.context.id,
+        label: input.context.label,
+        runtime: input.context.runtime,
+        details: input.details,
+        startedAt: input.context.startedAt,
+        ...(input.context.conversationId ? { conversationId: input.context.conversationId } : {}),
+        ...(input.runArtifacts.runId ? { runId: input.runArtifacts.runId } : {}),
+        ...(input.context.artifactPaths?.length ? { artifactPaths: input.context.artifactPaths } : {}),
+        traceSignature: input.runArtifacts.traceSignature,
+        toolStarts: input.runArtifacts.toolStarts,
+        toolArgsPreview: input.runArtifacts.toolArgsPreview,
+        textPreview: input.runArtifacts.textPreview,
+    });
+}
+function createStreamingFailureEvalResult(input) {
+    return createFailedEvalResult({
+        id: input.context.id,
+        label: input.context.label,
+        runtime: input.context.runtime,
+        details: `${input.details}${buildFailureSuffix(input.progress)}`,
+        startedAt: input.context.startedAt,
+        ...(input.context.conversationId ? { conversationId: input.context.conversationId } : {}),
+        ...(input.context.artifactPaths?.length ? { artifactPaths: input.context.artifactPaths } : {}),
+        toolStarts: input.progress.toolStarts,
+        textPreview: input.progress.textLength > 0
+            ? `${input.progress.textLength} characters streamed`
+            : undefined,
+    });
+}
+function createLiveEvalResultContext(input) {
+    return {
+        id: input.testCase.id,
+        label: input.testCase.label,
+        runtime: input.runtime,
+        startedAt: input.startedAt,
+        ...(input.conversationId ? { conversationId: input.conversationId } : {}),
+        ...(input.artifactPaths.length > 0 ? { artifactPaths: input.artifactPaths } : {}),
+    };
+}
+function buildLiveEvalRunBody(input) {
+    const customBody = typeof input.prepared?.metadata?.customBody === "string"
+        ? input.prepared.metadata.customBody
+        : null;
+    if (customBody) {
+        return JSON.parse(customBody);
+    }
+    return buildLiveEvalRequestBody({
+        testCaseId: input.testCase.id,
+        prompt: input.prepared?.prompt ?? input.testCase.prompt ?? "",
+        metadata: input.prepared?.metadata,
+        projectId: input.config.projectId && input.testCase.requireProject
+            ? input.config.projectId
+            : null,
+        ...(input.config.branchId ? { branchId: input.config.branchId } : {}),
+        ...(input.config.model ? { model: input.config.model } : {}),
+        ...(input.conversationId ? { conversationId: input.conversationId } : {}),
+        allowedTools: input.testCase.allowedTools,
+        forceRuntimeOverrides: input.testCase.forceRuntimeOverrides,
+        maxSteps: input.testCase.maxSteps,
+    });
+}
+async function resolveCompletedLiveEvalRun(input) {
+    const traceSignature = buildTraceSignature(input.run.eventTypes);
+    const runArtifacts = createLiveEvalRunArtifacts({
+        run: input.run,
+        runId: input.runId,
+        traceSignature,
+    });
+    const failure = await input.testCase.verify(input.run, input.prepared);
+    if (!failure && input.testCase.expectedEventSubsequence) {
+        if (!containsOrderedSubsequence(input.run.eventTypes, input.testCase.expectedEventSubsequence)) {
+            return createFailedRunEvalResult({
+                context: input.context,
+                details: `Expected AG-UI event subsequence ${input.testCase.expectedEventSubsequence.join(" -> ")}, got ${traceSignature}`,
+                runArtifacts,
+            });
+        }
+    }
+    if (failure) {
+        return createFailedRunEvalResult({
+            context: input.context,
+            details: failure,
+            runArtifacts,
+        });
+    }
+    return createPassedRunEvalResult({
+        context: input.context,
+        details: `OK: ${input.run.toolStarts.join(", ") || "no tools"} | ${input.run.text.slice(0, 140) || "no text"}`,
+        runArtifacts,
+    });
+}
+function extractRunId(run) {
+    for (const event of run.events) {
+        const runId = getStringField(event, "runId") ?? getStringField(event, "run_id");
+        if (runId) {
+            return runId;
+        }
+    }
+    return null;
+}
+export function hasFinished(run) {
+    return run.eventTypes.includes(agUiSseEventTypes.runFinished) && !run.runError;
+}
+export function containsSkillLoad(run, skillId) {
+    return run.toolStarts.includes("load_skill") && run.toolArgs.join("").includes(skillId);
+}
+export function countStepStartedEvents(run) {
+    return run.eventTypes.filter((eventType) => eventType === agUiSseEventTypes.stepStarted).length;
+}
+export function createLiveEvalCaseSupport(config) {
+    const fetchImpl = resolveFetch(config);
+    const log = config.log ?? console.log;
+    const { judgeLlm, withJudge } = createLiveEvalJudgeSupport(config);
+    async function verifyFileExists(input) {
+        if (!config.projectId || !config.readProjectFile) {
+            return null;
+        }
+        const file = await config.readProjectFile({
+            filePath: input.filePath,
+            requestTimeoutMs: config.requestTimeoutMs,
+        });
+        if (!file) {
+            return `${input.description ?? input.filePath}: file not found in project after task completed`;
+        }
+        if (!file.content || file.content.trim().length === 0) {
+            return `${input.description ?? input.filePath}: file exists but is empty`;
+        }
+        if (input.requiredContent) {
+            const missing = input.requiredContent.filter((keyword) => !file.content.toLowerCase().includes(keyword.toLowerCase()));
+            if (missing.length > 0) {
+                return `${input.description ?? input.filePath}: missing required content: ${missing.join(", ")}. Got: ${file.content.slice(0, 200)}`;
+            }
+        }
+        return null;
+    }
+    async function runEval(testCase, runtime) {
+        const startedAt = Date.now();
+        if (testCase.requireProject && !config.projectId) {
+            return createSkippedEvalResult({
+                id: testCase.id,
+                label: testCase.label,
+                runtime,
+                details: "Skipped because AG_UI_EVAL_PROJECT_ID is not set.",
+                startedAt,
+            });
+        }
+        const prepared = testCase.prepare
+            ? await testCase.prepare({
+                apiUrl: config.apiUrl,
+                authToken: config.authToken,
+                projectId: config.projectId,
+            })
+            : null;
+        const preparedConversationId = extractPreparedConversationId(prepared);
+        const preparedArtifactPaths = collectPreparedArtifactPaths(prepared);
+        const resultContext = createLiveEvalResultContext({
+            testCase,
+            runtime,
+            startedAt,
+            conversationId: preparedConversationId,
+            artifactPaths: preparedArtifactPaths,
+        });
+        try {
+            const sidecarCleanup = prepared?.startSidecar ? await prepared.startSidecar() : undefined;
+            const progressReporter = createLiveEvalProgressReporter({
+                caseId: testCase.id,
+                startedAt,
+                intervalMs: config.progressLogIntervalMs,
+                log,
+            });
+            const body = buildLiveEvalRunBody({
+                config,
+                testCase,
+                prepared,
+                conversationId: preparedConversationId,
+            });
+            try {
+                const response = await fetchImpl(config.endpoint, {
+                    method: "POST",
+                    headers: {
+                        "Content-Type": "application/json",
+                        Authorization: `Bearer ${config.authToken}`,
+                    },
+                    body: JSON.stringify(body),
+                    signal: AbortSignal.timeout(config.requestTimeoutMs),
+                });
+                log(`[stream] ${runtime}:${testCase.id} HTTP ${response.status}`);
+                const run = await parseSseResponse(response, {
+                    onProgress: progressReporter.update,
+                });
+                return resolveCompletedLiveEvalRun({
+                    testCase,
+                    run,
+                    prepared,
+                    context: resultContext,
+                    runId: extractRunId(run) ?? undefined,
+                });
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                return createStreamingFailureEvalResult({
+                    context: resultContext,
+                    details: message,
+                    progress: progressReporter.getSnapshot(),
+                });
+            }
+            finally {
+                progressReporter.stop();
+                await sidecarCleanup?.();
+            }
+        }
+        finally {
+            await prepared?.cleanup?.();
+        }
+    }
+    return {
+        judgeLlm,
+        runEval,
+        verifyFileExists,
+        withJudge,
+    };
+}
+export const liveEvalRunnerInternals = {
+    collectPreparedArtifactPaths,
+    createFailedRunEvalResult,
+    createLiveEvalRunArtifacts,
+    createPassedRunEvalResult,
+    createStreamingFailureEvalResult,
+    extractRunId,
+};

package/esm/src/utils/version-constant.d.ts

@@ -1,2 +1,2 @@
-export declare const VERSION = "0.1.522";
+export declare const VERSION = "0.1.523";
 //# sourceMappingURL=version-constant.d.ts.map

package/esm/src/utils/version-constant.js

@@ -1,3 +1,3 @@
 // Keep in sync with deno.json version.
 // scripts/release.ts updates this constant during releases.
-export const VERSION = "0.1.522";
+export const VERSION = "0.1.523";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "veryfront",
-  "version": "0.1.522",
+  "version": "0.1.523",
   "description": "The simplest way to build AI-powered apps",
   "keywords": [
     "react",

package/src/deno.js

@@ -1,6 +1,6 @@
 export default {
   "name": "veryfront",
-  "version": "0.1.522",
+  "version": "0.1.523",
   "license": "Apache-2.0",
   "nodeModulesDir": "auto",
   "workspace": [
@@ -296,9 +296,6 @@ export default {
     "strict": true,
     "noImplicitAny": true,
     "noUncheckedIndexedAccess": true,
-    "types": [
-      "npm:@types/react@19.2.14"
-    ],
     "lib": [
       "deno.window",
       "dom",
@@ -431,14 +428,5 @@ export default {
     "semiColons": true,
     "singleQuote": false,
     "proseWrap": "preserve"
-  },
-  "allowScripts": {
-    "allow": [
-      "npm:sharp@0.33.5",
-      "npm:onnxruntime-node@1.21.0"
-    ],
-    "deny": [
-      "npm:protobufjs@7.5.4"
-    ]
   }
 };

package/src/src/agent/testing/index.ts

@@ -27,17 +27,29 @@ export {
   buildProgressLine,
   buildRuntimePerformanceSummary,
   containsOrderedSubsequence,
+  containsSkillLoad,
+  countStepStartedEvents,
   createFailedEvalResult,
+  createLiveEvalCaseSupport,
   createPassedEvalResult,
   createPlainTextPdf,
   createSkippedEvalResult,
   hasEveryLiveEvalTag,
+  hasFinished,
+  type LiveEvalCase,
   type LiveEvalCaseMetadata,
   type LiveEvalCaseSelectionInput,
+  type LiveEvalContext,
+  type LiveEvalProjectFile,
+  type LiveEvalProjectFileReaderInput,
+  type LiveEvalRequestBody,
   type LiveEvalResultForPerformance,
   type LiveEvalResultForReport,
   type LiveEvalResultRecord,
+  type LiveEvalRunnerConfig,
+  liveEvalRunnerInternals,
   type LiveEvalRuntime,
+  type PreparedLiveEvalInput,
   resolveLiveEvalRequestedCaseIds,
   type RuntimePerformanceSummary,
   selectLiveEvalCases,

package/src/src/agent/testing/live-evals/index.ts

@@ -10,7 +10,11 @@ export {
   type LiveEvalRuntime,
   type RuntimePerformanceSummary,
 } from "./performance.js";
-export { buildLiveEvalRequestBody, type BuildLiveEvalRequestBodyInput } from "./request.js";
+export {
+  buildLiveEvalRequestBody,
+  type BuildLiveEvalRequestBodyInput,
+  type LiveEvalRequestBody,
+} from "./request.js";
 export {
   buildLiveEvalCaseTagSummary,
   buildLiveEvalRuntimeSummary,
@@ -28,3 +32,16 @@ export {
   createSkippedEvalResult,
   type LiveEvalResultRecord,
 } from "./result.js";
+export {
+  containsSkillLoad,
+  countStepStartedEvents,
+  createLiveEvalCaseSupport,
+  hasFinished,
+  type LiveEvalCase,
+  type LiveEvalContext,
+  type LiveEvalProjectFile,
+  type LiveEvalProjectFileReaderInput,
+  type LiveEvalRunnerConfig,
+  liveEvalRunnerInternals,
+  type PreparedLiveEvalInput,
+} from "./runner.js";

package/src/src/agent/testing/live-evals/request.ts

@@ -1,4 +1,20 @@
 import * as dntShim from "../../../../_dnt.shims.js";
+export interface LiveEvalRequestBody {
+  threadId: string;
+  runId: string;
+  state: Record<string, string>;
+  tools: unknown[];
+  context: unknown[];
+  forwardedProps?: {
+    veryfront: Record<string, unknown>;
+  };
+  messages: Array<{
+    id: string;
+    role: "user";
+    content: string;
+  }>;
+}
+
 export interface BuildLiveEvalRequestBodyInput {
   testCaseId: string;
   prompt: string;
@@ -12,7 +28,9 @@ export interface BuildLiveEvalRequestBodyInput {
   maxSteps?: number;
 }
 
-export function buildLiveEvalRequestBody(input: BuildLiveEvalRequestBodyInput) {
+export function buildLiveEvalRequestBody(
+  input: BuildLiveEvalRequestBodyInput,
+): LiveEvalRequestBody {
   const veryfront: Record<string, unknown> = {};
   if (input.projectId) {
     veryfront.projectId = input.projectId;