veryfront 0.1.876 → 0.1.877

package/esm/deno.d.ts

@@ -24,8 +24,8 @@ declare namespace _default {
         "./mdx": string;
         "./agent": string;
         "./agent/identity": string;
-        "./agent/testing": string;
         "./eval": string;
+        "./eval/agent-service": string;
         "./channels/control-plane": string;
         "./channels/invoke": string;
         "./tool": string;
@@ -93,6 +93,7 @@ declare namespace _default {
         "veryfront/embedding": string;
         "veryfront/runs": string;
         "veryfront/eval": string;
+        "veryfront/eval/agent-service": string;
         "veryfront/agent/identity": string;
         "veryfront/agent/react": string;
         "veryfront/agent/middleware": string;
@@ -138,7 +139,6 @@ declare namespace _default {
         "veryfront/provider/shared": string;
         "veryfront/tool/schema": string;
         "veryfront/agent/composition": string;
-        "veryfront/agent/testing": string;
         "veryfront/extensions": string;
         "veryfront/testing": string;
         "veryfront/testing/assert": string;
@@ -173,6 +173,7 @@ declare namespace _default {
         "#veryfront/data": string;
         "#veryfront/errors": string;
         "#veryfront/eval": string;
+        "#veryfront/eval/agent-service": string;
         "#veryfront/html": string;
         "#veryfront/mcp": string;
         "#veryfront/middleware": string;

package/esm/deno.js

@@ -1,6 +1,6 @@
 export default {
     "name": "veryfront",
-    "version": "0.1.876",
+    "version": "0.1.877",
     "license": "Apache-2.0",
     "nodeModulesDir": "auto",
     "minimumDependencyAge": {
@@ -79,8 +79,8 @@ export default {
         "./mdx": "./src/mdx/index.ts",
         "./agent": "./src/agent/index.ts",
         "./agent/identity": "./src/agent/identity-contracts.ts",
-        "./agent/testing": "./src/agent/testing/index.ts",
         "./eval": "./src/eval/index.ts",
+        "./eval/agent-service": "./src/eval/agent-service.ts",
         "./channels/control-plane": "./src/channels/control-plane.ts",
         "./channels/invoke": "./src/channels/invoke.ts",
         "./tool": "./src/tool/index.ts",
@@ -148,6 +148,7 @@ export default {
         "veryfront/embedding": "./src/embedding/index.ts",
         "veryfront/runs": "./src/runs/index.ts",
         "veryfront/eval": "./src/eval/index.ts",
+        "veryfront/eval/agent-service": "./src/eval/agent-service.ts",
         "veryfront/agent/identity": "./src/agent/identity-contracts.ts",
         "veryfront/agent/react": "./src/agent/react/index.ts",
         "veryfront/agent/middleware": "./src/agent/middleware/index.ts",
@@ -193,7 +194,6 @@ export default {
         "veryfront/provider/shared": "./src/provider/shared/index.ts",
         "veryfront/tool/schema": "./src/tool/schema/index.ts",
         "veryfront/agent/composition": "./src/agent/composition/index.ts",
-        "veryfront/agent/testing": "./src/agent/testing/index.ts",
         "veryfront/extensions": "./src/extensions/index.ts",
         "veryfront/testing": "./src/testing/index.ts",
         "veryfront/testing/assert": "./src/testing/assert.ts",
@@ -228,6 +228,7 @@ export default {
         "#veryfront/data": "./src/data/index.ts",
         "#veryfront/errors": "./src/errors/index.ts",
         "#veryfront/eval": "./src/eval/index.ts",
+        "#veryfront/eval/agent-service": "./src/eval/agent-service.ts",
         "#veryfront/html": "./src/html/index.ts",
         "#veryfront/mcp": "./src/mcp/index.ts",
         "#veryfront/middleware": "./src/middleware/index.ts",

package/esm/src/eval/agent-service.d.ts

@@ -0,0 +1,88 @@
+import "../../_dnt.polyfills.js";
+import { type AgUiSseProgressSnapshot } from "../agent/index.js";
+import type { EvalAgentAdapter, EvalAgentAdapterContext } from "./types.js";
+/** Default local AG-UI endpoint used by agent-service evals. */
+export declare const DEFAULT_AGENT_SERVICE_EVAL_ENDPOINT = "http://127.0.0.1:3001/api/ag-ui";
+/** Environment input accepted by agent-service eval helpers. */
+export type AgentServiceEvalEnvironmentInput = Record<string, string | number | undefined>;
+/** Resolved environment values for live agent-service evals. */
+export interface AgentServiceEvalEnvironment {
+    endpoint: string;
+    authToken: string;
+    apiUrl: string;
+    projectId?: string;
+    branchId?: string;
+    model?: string;
+}
+/** Preflight result for a live agent-service eval environment. */
+export interface AgentServiceEvalEnvironmentPreflightResult {
+    ok: boolean;
+    resolvedApiUrl: string;
+    messages: string[];
+}
+/** Veryfront forwarded props included in an AG-UI eval request. */
+export interface AgentServiceEvalForwardedProps {
+    projectId?: string;
+    conversationId?: string;
+    branchId?: string;
+    model?: string;
+    runtimeOverrides?: {
+        allowedTools?: string[];
+        maxSteps?: number;
+    };
+}
+/** Input accepted by `buildAgentServiceEvalRequestBody`. */
+export interface BuildAgentServiceEvalRequestBodyInput {
+    exampleId: string;
+    input: unknown;
+    metadata?: Record<string, unknown>;
+    projectId?: string | null;
+    conversationId?: string | null;
+    branchId?: string | null;
+    model?: string | null;
+    allowedTools?: string[];
+    forceRuntimeOverrides?: boolean;
+    maxSteps?: number;
+}
+/** AG-UI request body sent to an agent-service endpoint. */
+export interface AgentServiceEvalRequestBody {
+    threadId: string;
+    runId: string;
+    state: Record<string, unknown>;
+    tools: [];
+    context: [];
+    forwardedProps?: {
+        veryfront: AgentServiceEvalForwardedProps;
+    };
+    messages: Array<{
+        id: string;
+        role: "user";
+        content: string;
+    }>;
+}
+/** Configuration for the live agent-service eval adapter. */
+export interface AgentServiceEvalAdapterConfig {
+    endpoint?: string;
+    authToken: string;
+    projectId?: string | null;
+    conversationId?: string | null;
+    branchId?: string | null;
+    model?: string | null;
+    allowedTools?: string[];
+    forceRuntimeOverrides?: boolean;
+    maxSteps?: number;
+    requestTimeoutMs?: number;
+    progressThrottleMs?: number;
+    fetch?: (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+    now?: () => number;
+    onProgress?: (snapshot: AgUiSseProgressSnapshot, context: EvalAgentAdapterContext) => void;
+}
+/** Resolve environment values for live agent-service eval execution. */
+export declare function resolveAgentServiceEvalEnvironment(env?: AgentServiceEvalEnvironmentInput): AgentServiceEvalEnvironment;
+/** Evaluate whether the required live agent-service eval environment is present. */
+export declare function evaluateAgentServiceEvalEnvironment(env?: AgentServiceEvalEnvironmentInput, resolvedApiUrl?: string): AgentServiceEvalEnvironmentPreflightResult;
+/** Build the AG-UI request body for a single eval example. */
+export declare function buildAgentServiceEvalRequestBody(input: BuildAgentServiceEvalRequestBodyInput): AgentServiceEvalRequestBody;
+/** Create an `EvalAgentAdapter` that executes examples against an AG-UI agent-service endpoint. */
+export declare function createAgentServiceEvalAdapter(config: AgentServiceEvalAdapterConfig): EvalAgentAdapter;
+//# sourceMappingURL=agent-service.d.ts.map

package/esm/src/eval/agent-service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-service.d.ts","sourceRoot":"","sources":["../../../src/src/eval/agent-service.ts"],"names":[],"mappings":"AAAA,OAAO,yBAAyB,CAAC;AAEjC,OAAO,EAEL,KAAK,uBAAuB,EAK7B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,gBAAgB,EAChB,uBAAuB,EAGxB,MAAM,YAAY,CAAC;AAEpB,gEAAgE;AAChE,eAAO,MAAM,mCAAmC,oCAAoC,CAAC;AAErF,gEAAgE;AAChE,MAAM,MAAM,gCAAgC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC,CAAC;AAE3F,gEAAgE;AAChE,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,kEAAkE;AAClE,MAAM,WAAW,0CAA0C;IACzD,EAAE,EAAE,OAAO,CAAC;IACZ,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,mEAAmE;AACnE,MAAM,WAAW,8BAA8B;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE;QACjB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED,4DAA4D;AAC5D,MAAM,WAAW,qCAAqC;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,OAAO,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,4DAA4D;AAC5D,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,KAAK,EAAE,EAAE,CAAC;IACV,OAAO,EAAE,EAAE,CAAC;IACZ,cAAc,CAAC,EAAE;QACf,SAAS,EAAE,8BAA8B,CAAC;KAC3C,CAAC;IACF,QAAQ,EAAE,KAAK,CAAC;QACd,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AAED,6DAA6D;AAC7D,MAAM,WAAW,6BAA6B;IAC5C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjF,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,uBAAuB,EAAE,OAAO,EAAE,uBAAuB,KAAK,IAAI,CAAC;CAC5F;AAmKD,wEAAwE;AACxE,wBAAgB,kCAAkC,CAChD,GAAG,GAAE,gCAAqC,GACzC,2BAA2B,CAe7B;AAED,oFAAoF;AACpF,wBAAgB,mCAAmC,CACjD,GAAG,GAAE,gCAAqC,EAC1C,cAAc,SAAiD,GAC9D,0CAA0C,CAe5C;AAED,8DAA8D;AAC9D,wBAAgB,gCAAgC,CAC9C,KAAK,EAAE,qCAAqC,GAC3C,2BAA2B,CAyB7B;AAED,mGAAmG;AACnG,wBAAgB,6BAA6B,CAC3C,MAAM,EAAE,6BAA6B,GACpC,gBAAgB,CAoElB"}

package/esm/src/eval/agent-service.js

@@ -0,0 +1,274 @@
+import "../../_dnt.polyfills.js";
+import * as dntShim from "../../_dnt.shims.js";
+import { agUiSseEventTypes, getAgUiSseStringField, parseAgentServiceConfig, parseAgUiSseResponse, } from "../agent/index.js";
+/** Default local AG-UI endpoint used by agent-service evals. */
+export const DEFAULT_AGENT_SERVICE_EVAL_ENDPOINT = "http://127.0.0.1:3001/api/ag-ui";
+function readString(value) {
+    return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+function readNumber(value) {
+    return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+function readStringArray(value) {
+    return Array.isArray(value) && value.every((entry) => typeof entry === "string")
+        ? [...value]
+        : undefined;
+}
+function isRecord(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function stringifyPromptInput(input) {
+    if (typeof input === "string")
+        return input;
+    if (isRecord(input)) {
+        const prompt = readString(input.prompt) ?? readString(input.message) ?? readString(input.input);
+        if (prompt)
+            return prompt;
+    }
+    return JSON.stringify(input);
+}
+function getInputMetadata(input) {
+    if (!isRecord(input) || !isRecord(input.metadata))
+        return {};
+    return { ...input.metadata };
+}
+function getRequestOverrides(input) {
+    const record = isRecord(input.input) ? input.input : {};
+    return {
+        projectId: input.projectId ?? readString(record.projectId) ?? null,
+        conversationId: input.conversationId ?? readString(record.conversationId) ?? null,
+        branchId: input.branchId ?? readString(record.branchId) ?? null,
+        model: input.model ?? readString(record.model) ?? null,
+        allowedTools: input.allowedTools ?? readStringArray(record.allowedTools),
+        forceRuntimeOverrides: input.forceRuntimeOverrides ?? record.forceRuntimeOverrides === true,
+        maxSteps: input.maxSteps ?? readNumber(record.maxSteps),
+    };
+}
+function createVeryfrontForwardedProps(input) {
+    const overrides = getRequestOverrides(input);
+    const veryfront = {};
+    if (overrides.projectId)
+        veryfront.projectId = overrides.projectId;
+    if (overrides.conversationId)
+        veryfront.conversationId = overrides.conversationId;
+    if (overrides.branchId)
+        veryfront.branchId = overrides.branchId;
+    if (overrides.model)
+        veryfront.model = overrides.model;
+    const shouldForwardRuntimeOverrides = !!overrides.allowedTools ||
+        overrides.forceRuntimeOverrides ||
+        overrides.maxSteps !== undefined;
+    if (shouldForwardRuntimeOverrides) {
+        veryfront.runtimeOverrides = {
+            ...(overrides.allowedTools ? { allowedTools: overrides.allowedTools } : {}),
+            ...(overrides.maxSteps !== undefined ? { maxSteps: overrides.maxSteps } : {}),
+        };
+    }
+    return Object.keys(veryfront).length > 0 ? veryfront : null;
+}
+function createHeaders(authToken) {
+    return {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${authToken}`,
+    };
+}
+function getNow(config) {
+    return config.now?.() ?? Date.now();
+}
+function stringifyError(value) {
+    if (typeof value === "string" && value.length > 0)
+        return value;
+    if (isRecord(value)) {
+        const message = readString(value.message) ?? readString(value.error);
+        if (message)
+            return message;
+    }
+    try {
+        return JSON.stringify(value);
+    }
+    catch {
+        return String(value);
+    }
+}
+function getToolResultError(event) {
+    return stringifyError(event.result) ?? stringifyError(event.content) ?? "Tool call failed";
+}
+function createToolCalls(events) {
+    const toolCalls = new Map();
+    for (const [index, event] of events.entries()) {
+        const type = getAgUiSseStringField(event, "type");
+        if (type === agUiSseEventTypes.toolCallStart) {
+            const name = getAgUiSseStringField(event, "toolCallName");
+            if (!name)
+                continue;
+            const id = getAgUiSseStringField(event, "toolCallId");
+            const key = id ?? `name:${name}`;
+            const existing = toolCalls.get(key);
+            toolCalls.set(key, {
+                name,
+                status: existing?.status ?? "ok",
+                ...(existing?.error ? { error: existing.error } : {}),
+                ...(existing?.metadata ? { metadata: existing.metadata } : {}),
+            });
+            continue;
+        }
+        if (type === agUiSseEventTypes.toolCallResult) {
+            const id = getAgUiSseStringField(event, "toolCallId");
+            const toolName = getAgUiSseStringField(event, "toolCallName");
+            const key = id ?? (toolName ? `name:${toolName}` : `result:${index}`);
+            const existing = toolCalls.get(key);
+            const failed = event.isError === true;
+            toolCalls.set(key, {
+                name: existing?.name ?? toolName ?? "tool",
+                status: failed ? "error" : existing?.status ?? "ok",
+                ...(failed ? { error: getToolResultError(event) } : {}),
+                ...(existing?.metadata ? { metadata: existing.metadata } : {}),
+            });
+        }
+    }
+    return [...toolCalls.values()];
+}
+function createRunOutput(run) {
+    return {
+        text: run.text,
+        agUi: {
+            responseStatus: run.responseStatus,
+            eventTypes: run.eventTypes,
+            runError: run.runError,
+        },
+    };
+}
+function createRequestInit(config, body) {
+    return {
+        method: "POST",
+        headers: createHeaders(config.authToken),
+        body: JSON.stringify(body),
+        ...(config.requestTimeoutMs ? { signal: AbortSignal.timeout(config.requestTimeoutMs) } : {}),
+    };
+}
+/** Resolve environment values for live agent-service eval execution. */
+export function resolveAgentServiceEvalEnvironment(env = {}) {
+    return {
+        endpoint: typeof env.AG_UI_EVAL_ENDPOINT === "string"
+            ? env.AG_UI_EVAL_ENDPOINT
+            : DEFAULT_AGENT_SERVICE_EVAL_ENDPOINT,
+        authToken: typeof env.VERYFRONT_TOKEN === "string" ? env.VERYFRONT_TOKEN : "",
+        apiUrl: typeof env.VERYFRONT_API_URL === "string"
+            ? env.VERYFRONT_API_URL
+            : parseAgentServiceConfig(env).VERYFRONT_API_URL,
+        ...(typeof env.AG_UI_EVAL_PROJECT_ID === "string"
+            ? { projectId: env.AG_UI_EVAL_PROJECT_ID }
+            : {}),
+        ...(typeof env.AG_UI_EVAL_BRANCH_ID === "string" ? { branchId: env.AG_UI_EVAL_BRANCH_ID } : {}),
+        ...(typeof env.AG_UI_EVAL_MODEL === "string" ? { model: env.AG_UI_EVAL_MODEL } : {}),
+    };
+}
+/** Evaluate whether the required live agent-service eval environment is present. */
+export function evaluateAgentServiceEvalEnvironment(env = {}, resolvedApiUrl = parseAgentServiceConfig(env).VERYFRONT_API_URL) {
+    const messages = [`Resolved VERYFRONT_API_URL: ${resolvedApiUrl}`];
+    let hasBlockers = false;
+    if (typeof env.VERYFRONT_TOKEN !== "string" || env.VERYFRONT_TOKEN.length === 0) {
+        hasBlockers = true;
+        messages.push("BLOCKER: VERYFRONT_TOKEN is missing");
+    }
+    if (typeof env.AG_UI_EVAL_PROJECT_ID !== "string" || env.AG_UI_EVAL_PROJECT_ID.length === 0) {
+        hasBlockers = true;
+        messages.push("BLOCKER: AG_UI_EVAL_PROJECT_ID is missing");
+    }
+    messages.push(`Agent-service eval preflight: ${hasBlockers ? "FAIL" : "PASS"}`);
+    return { ok: !hasBlockers, resolvedApiUrl, messages };
+}
+/** Build the AG-UI request body for a single eval example. */
+export function buildAgentServiceEvalRequestBody(input) {
+    const veryfront = createVeryfrontForwardedProps(input);
+    const metadata = {
+        ...getInputMetadata(input.input),
+        ...(input.metadata ?? {}),
+    };
+    return {
+        threadId: dntShim.crypto.randomUUID(),
+        runId: `eval-run-${dntShim.crypto.randomUUID()}`,
+        state: {
+            evalCase: input.exampleId,
+            ...metadata,
+        },
+        tools: [],
+        context: [],
+        ...(veryfront ? { forwardedProps: { veryfront } } : {}),
+        messages: [
+            {
+                id: dntShim.crypto.randomUUID(),
+                role: "user",
+                content: stringifyPromptInput(input.input),
+            },
+        ],
+    };
+}
+/** Create an `EvalAgentAdapter` that executes examples against an AG-UI agent-service endpoint. */
+export function createAgentServiceEvalAdapter(config) {
+    const requestFetch = config.fetch ?? fetch;
+    const endpoint = config.endpoint ?? DEFAULT_AGENT_SERVICE_EVAL_ENDPOINT;
+    return async (context) => {
+        const started = getNow(config);
+        const body = buildAgentServiceEvalRequestBody({
+            exampleId: context.example.id,
+            input: context.example.input,
+            metadata: context.example.metadata,
+            projectId: config.projectId,
+            conversationId: config.conversationId,
+            branchId: config.branchId,
+            model: config.model,
+            allowedTools: config.allowedTools,
+            forceRuntimeOverrides: config.forceRuntimeOverrides,
+            maxSteps: config.maxSteps,
+        });
+        try {
+            const parseOptions = {
+                ...(config.progressThrottleMs !== undefined
+                    ? { progressThrottleMs: config.progressThrottleMs }
+                    : {}),
+                ...(config.onProgress
+                    ? { onProgress: (snapshot) => config.onProgress?.(snapshot, context) }
+                    : {}),
+            };
+            const response = await requestFetch(endpoint, createRequestInit(config, body));
+            const run = await parseAgUiSseResponse(response, parseOptions);
+            const completed = response.ok && run.runError === null &&
+                run.eventTypes.includes(agUiSseEventTypes.runFinished);
+            const output = createRunOutput(run);
+            return {
+                text: run.text,
+                output,
+                trace: {
+                    events: run.events,
+                    toolCalls: createToolCalls(run.events),
+                },
+                durationMs: getNow(config) - started,
+                completed,
+                ...(!completed
+                    ? { error: run.runError ?? `AG-UI response failed with status ${response.status}` }
+                    : {}),
+            };
+        }
+        catch (error) {
+            return {
+                text: "",
+                output: {
+                    text: "",
+                    agUi: {
+                        responseStatus: 0,
+                        eventTypes: [],
+                        runError: error instanceof Error ? error.message : String(error),
+                    },
+                },
+                trace: {
+                    events: [],
+                    toolCalls: [],
+                },
+                durationMs: getNow(config) - started,
+                completed: false,
+                error: error instanceof Error ? error.message : String(error),
+            };
+        }
+    };
+}

package/esm/src/eval/index.d.ts

@@ -18,6 +18,28 @@
  *   ],
  * });
  * ```
+ *
+ * @example Live agent-service eval
+ * ```ts
+ * import { datasets, evalAgent, metrics, runEval } from "veryfront/eval";
+ * import { createAgentServiceEvalAdapter } from "veryfront/eval/agent-service";
+ *
+ * const definition = evalAgent({
+ *   target: "agent:veryfront",
+ *   dataset: datasets.inline([{ id: "smoke", input: "List project files." }]),
+ *   metrics: [metrics.agent.noFailedTools().gate()],
+ * });
+ *
+ * const report = await runEval(definition, {
+ *   adapters: {
+ *     agent: createAgentServiceEvalAdapter({
+ *       endpoint: "http://127.0.0.1:3001/api/ag-ui",
+ *       authToken: "<TOKEN>",
+ *       projectId: "<PROJECT_ID>",
+ *     }),
+ *   },
+ * });
+ * ```
  */
 import "../../_dnt.polyfills.js";
 export { datasets } from "./datasets.js";

package/esm/src/eval/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EACL,wBAAwB,EACxB,0BAA0B,EAC1B,gBAAgB,EAChB,2BAA2B,EAC3B,wBAAwB,EACxB,4BAA4B,EAC5B,6BAA6B,GAC9B,MAAM,aAAa,CAAC;AAErB,YAAY,EAAE,cAAc,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAChG,YAAY,EACV,gBAAgB,EAChB,uBAAuB,EACvB,sBAAsB,EACtB,cAAc,EACd,gBAAgB,EAChB,WAAW,EACX,sBAAsB,EACtB,cAAc,EACd,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,eAAe,EACf,UAAU,EACV,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,UAAU,EACV,UAAU,EACV,iBAAiB,EACjB,YAAY,EACZ,UAAU,EACV,cAAc,EACd,YAAY,EACZ,SAAS,EACT,SAAS,EACT,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,+BAA+B,EAC/B,iBAAiB,EACjB,OAAO,EACP,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EACL,wBAAwB,EACxB,0BAA0B,EAC1B,gBAAgB,EAChB,2BAA2B,EAC3B,wBAAwB,EACxB,4BAA4B,EAC5B,6BAA6B,GAC9B,MAAM,aAAa,CAAC;AAErB,YAAY,EAAE,cAAc,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAChG,YAAY,EACV,gBAAgB,EAChB,uBAAuB,EACvB,sBAAsB,EACtB,cAAc,EACd,gBAAgB,EAChB,WAAW,EACX,sBAAsB,EACtB,cAAc,EACd,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,eAAe,EACf,UAAU,EACV,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,UAAU,EACV,UAAU,EACV,iBAAiB,EACjB,YAAY,EACZ,UAAU,EACV,cAAc,EACd,YAAY,EACZ,SAAS,EACT,SAAS,EACT,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,+BAA+B,EAC/B,iBAAiB,EACjB,OAAO,EACP,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,aAAa,CAAC"}

package/esm/src/eval/index.js

@@ -18,6 +18,28 @@
  *   ],
  * });
  * ```
+ *
+ * @example Live agent-service eval
+ * ```ts
+ * import { datasets, evalAgent, metrics, runEval } from "veryfront/eval";
+ * import { createAgentServiceEvalAdapter } from "veryfront/eval/agent-service";
+ *
+ * const definition = evalAgent({
+ *   target: "agent:veryfront",
+ *   dataset: datasets.inline([{ id: "smoke", input: "List project files." }]),
+ *   metrics: [metrics.agent.noFailedTools().gate()],
+ * });
+ *
+ * const report = await runEval(definition, {
+ *   adapters: {
+ *     agent: createAgentServiceEvalAdapter({
+ *       endpoint: "http://127.0.0.1:3001/api/ag-ui",
+ *       authToken: "<TOKEN>",
+ *       projectId: "<PROJECT_ID>",
+ *     }),
+ *   },
+ * });
+ * ```
  */
 import "../../_dnt.polyfills.js";
 export { datasets } from "./datasets.js";

package/esm/src/utils/version-constant.d.ts

@@ -1,3 +1,3 @@
 /** Shared version value. */
-export declare const VERSION = "0.1.876";
+export declare const VERSION = "0.1.877";
 //# sourceMappingURL=version-constant.d.ts.map

package/esm/src/utils/version-constant.js

@@ -1,4 +1,4 @@
 // Keep in sync with deno.json version.
 // scripts/release.ts updates this constant during releases.
 /** Shared version value. */
-export const VERSION = "0.1.876";
+export const VERSION = "0.1.877";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "veryfront",
-  "version": "0.1.876",
+  "version": "0.1.877",
   "description": "The simplest way to build AI-powered apps",
   "keywords": [
     "react",
@@ -77,14 +77,14 @@
       "import": "./esm/src/agent/identity-contracts.js",
       "types": "./esm/src/agent/identity-contracts.d.ts"
     },
-    "./agent/testing": {
-      "import": "./esm/src/agent/testing/index.js",
-      "types": "./esm/src/agent/testing/index.d.ts"
-    },
     "./eval": {
       "import": "./esm/src/eval/index.js",
       "types": "./esm/src/eval/index.d.ts"
     },
+    "./eval/agent-service": {
+      "import": "./esm/src/eval/agent-service.js",
+      "types": "./esm/src/eval/agent-service.d.ts"
+    },
     "./channels/control-plane": {
       "import": "./esm/src/channels/control-plane.js",
       "types": "./esm/src/channels/control-plane.d.ts"

package/esm/src/agent/testing/agent-tester.d.ts

@@ -1,53 +0,0 @@
-import type { Agent, AgentResponse, Message } from "../types.js";
-/** Public API contract for test case. */
-export interface TestCase {
-    /** Test name */
-    name: string;
-    /** Input to agent */
-    input: string | Message[];
-    /** Expected output pattern (regex or string) */
-    expected?: RegExp | string;
-    /** Expected tool calls */
-    expectToolCalls?: string[];
-    /** Maximum execution time (ms) */
-    timeout?: number;
-    /** Custom validator */
-    validate?: (response: AgentResponse) => boolean | Promise<boolean>;
-}
-/** Result returned from test. */
-export interface TestResult {
-    /** Test case name */
-    name: string;
-    /** Pass/fail */
-    passed: boolean;
-    /** Response from agent */
-    response?: AgentResponse;
-    /** Error if test failed */
-    error?: string;
-    /** Execution time */
-    executionTime: number;
-    /** Tool calls made */
-    toolCalls: string[];
-}
-/** Public API contract for test suite. */
-export interface TestSuite {
-    /** Suite name */
-    name: string;
-    /** Test results */
-    results: TestResult[];
-    /** Overall pass/fail */
-    passed: boolean;
-    /** Total execution time */
-    totalTime: number;
-}
-/** Test agent helper. */
-export declare function testAgent(agent: Agent, testCases: TestCase[]): Promise<TestSuite>;
-/** Print test results helper. */
-export declare function printTestResults(suite: TestSuite): void;
-/** Assert contains helper. */
-export declare function assertContains(response: AgentResponse, text: string): boolean;
-/** Assert tool called helper. */
-export declare function assertToolCalled(response: AgentResponse, toolName: string): boolean;
-/** Assert completed helper. */
-export declare function assertCompleted(response: AgentResponse): boolean;
-//# sourceMappingURL=agent-tester.d.ts.map

package/esm/src/agent/testing/agent-tester.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"agent-tester.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/testing/agent-tester.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEjE,yCAAyC;AACzC,MAAM,WAAW,QAAQ;IACvB,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IAEb,qBAAqB;IACrB,KAAK,EAAE,MAAM,GAAG,OAAO,EAAE,CAAC;IAE1B,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAE3B,0BAA0B;IAC1B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAE3B,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uBAAuB;IACvB,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACpE;AAED,iCAAiC;AACjC,MAAM,WAAW,UAAU;IACzB,qBAAqB;IACrB,IAAI,EAAE,MAAM,CAAC;IAEb,gBAAgB;IAChB,MAAM,EAAE,OAAO,CAAC;IAEhB,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,aAAa,CAAC;IAEzB,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,qBAAqB;IACrB,aAAa,EAAE,MAAM,CAAC;IAEtB,sBAAsB;IACtB,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,0CAA0C;AAC1C,MAAM,WAAW,SAAS;IACxB,iBAAiB;IACjB,IAAI,EAAE,MAAM,CAAC;IAEb,mBAAmB;IACnB,OAAO,EAAE,UAAU,EAAE,CAAC;IAEtB,wBAAwB;IACxB,MAAM,EAAE,OAAO,CAAC;IAEhB,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,yBAAyB;AACzB,wBAAsB,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAWvF;AA6FD,iCAAiC;AACjC,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,SAAS,GAAG,IAAI,CAmBvD;AAED,8BAA8B;AAC9B,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAE7E;AAED,iCAAiC;AACjC,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAEnF;AAED,+BAA+B;AAC/B,wBAAgB,eAAe,CAAC,QAAQ,EAAE,aAAa,GAAG,OAAO,CAEhE"}