veryfront 0.1.870 → 0.1.872

package/esm/cli/commands/eval/command-help.d.ts

@@ -0,0 +1,3 @@
+import type { CommandHelp } from "../../help/types.js";
+export declare const evalHelp: CommandHelp;
+//# sourceMappingURL=command-help.d.ts.map

package/esm/cli/commands/eval/command-help.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"command-help.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command-help.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,eAAO,MAAM,QAAQ,EAAE,WAyCtB,CAAC"}

package/esm/cli/commands/eval/command-help.js

@@ -0,0 +1,42 @@
+export const evalHelp = {
+    name: "eval",
+    category: "ai",
+    description: "Discover and run eval definitions",
+    usage: "veryfront eval [eval-id] [options]",
+    options: [
+        {
+            flag: "-l, --list",
+            description: "List discovered evals",
+        },
+        {
+            flag: "--dataset-base <path>",
+            description: "Base directory for JSON and JSONL datasets",
+        },
+        {
+            flag: "--report <path>",
+            description: "Write the raw eval report JSON to a file",
+        },
+        {
+            flag: "--junit <path>",
+            description: "Write a JUnit XML report to a file",
+        },
+        {
+            flag: "--model <provider/model>",
+            description: "Override the target agent model",
+        },
+        {
+            flag: "--max-output-tokens <count>",
+            description: "Limit target agent output tokens",
+        },
+        {
+            flag: "--debug",
+            description: "Show discovery warnings",
+        },
+    ],
+    examples: [
+        "veryfront eval --list",
+        "veryfront eval deep-research",
+        "veryfront eval eval:deep-research --report .veryfront/evals/deep-research.json --junit .veryfront/evals/deep-research.xml",
+        "veryfront eval deep-research --json",
+    ],
+};

package/esm/cli/commands/eval/command.d.ts

@@ -0,0 +1,21 @@
+import type { EvalReport } from "../../../src/eval/index.js";
+import type { EvalArgs } from "./handler.js";
+export interface EvalOptions extends EvalArgs {
+}
+type CliEvalSummary = {
+    runId: string;
+    evalId: string;
+    target: string;
+    records: number;
+    passed: number;
+    failed: number;
+    passRate: number;
+    metrics: EvalReport["summary"]["metrics"];
+};
+export declare function normalizeEvalCliId(id: string): string;
+export declare function normalizeEvalInputForAgent(input: unknown): string;
+export declare function summarizeReportForCli(report: EvalReport): CliEvalSummary;
+export declare function createJunitXml(report: EvalReport): string;
+export declare function evalCommand(options: EvalOptions): Promise<void>;
+export {};
+//# sourceMappingURL=command.d.ts.map

package/esm/cli/commands/eval/command.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAKV,UAAU,EAEX,MAAM,4BAA4B,CAAC;AAWpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAiDF,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAmHD,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CA6FrE"}

package/esm/cli/commands/eval/command.js

@@ -0,0 +1,289 @@
+/**
+ * Eval command - Discover and run eval definitions from the evals/ directory.
+ */
+import * as dntShim from "../../../_dnt.shims.js";
+import { dirname, relative } from "../../../deps/jsr.io/@std/path/1.1.4/mod.js";
+import { createProjectDiscoveryConfig, discoverAll } from "../../../src/discovery/index.js";
+import { discoverEvals, runEval } from "../../../src/eval/index.js";
+import { cliLogger, exitProcess } from "../../utils/index.js";
+import { createErrorEnvelope, createSuccessEnvelope, isJsonMode, outputJson, } from "../../shared/json-output.js";
+import { withProjectSourceContext } from "../../shared/project-source-context.js";
+function xmlEscape(value) {
+    return value
+        .replaceAll("&", "&")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;")
+        .replaceAll("'", "&apos;");
+}
+function stripFileProtocol(path) {
+    if (!path.startsWith("file://"))
+        return path;
+    return decodeURIComponent(new URL(path).pathname);
+}
+function displaySourcePath(filePath, projectDir) {
+    const normalized = stripFileProtocol(filePath);
+    if (normalized.startsWith(projectDir)) {
+        return relative(projectDir, normalized);
+    }
+    return normalized;
+}
+function blockingResults(record) {
+    return [...(record.metrics ?? []), ...(record.checks ?? [])].filter((result) => !result.skipped && result.pass === false &&
+        (result.severity === "gate" || result.severity === "budget"));
+}
+function skippedResults(record) {
+    return [...(record.metrics ?? []), ...(record.checks ?? [])].filter((result) => result.skipped);
+}
+function testcaseName(record) {
+    return `${record.exampleId}#${record.repetition}`;
+}
+function failureMessage(result) {
+    return `${result.name} failed`;
+}
+function failureBody(result) {
+    if (result.explanation)
+        return result.explanation;
+    if (result.evidence)
+        return JSON.stringify(result.evidence);
+    return failureMessage(result);
+}
+export function normalizeEvalCliId(id) {
+    return id.startsWith("eval:") ? id : `eval:${id}`;
+}
+export function normalizeEvalInputForAgent(input) {
+    if (typeof input === "string")
+        return input;
+    if (input && typeof input === "object") {
+        const record = input;
+        for (const key of ["prompt", "question", "input"]) {
+            if (typeof record[key] === "string")
+                return record[key];
+        }
+    }
+    return JSON.stringify(input);
+}
+export function summarizeReportForCli(report) {
+    return {
+        runId: report.runId,
+        evalId: report.definitionId,
+        target: report.target,
+        records: report.summary.records,
+        passed: report.summary.passed,
+        failed: report.summary.failed,
+        passRate: report.summary.passRate,
+        metrics: report.summary.metrics,
+    };
+}
+export function createJunitXml(report) {
+    const skipped = report.records.reduce((count, record) => count + skippedResults(record).length, 0);
+    const lines = [
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        `<testsuite name="${xmlEscape(report.definitionId)}" tests="${report.summary.records}" failures="${report.summary.failed}" skipped="${skipped}">`,
+    ];
+    for (const record of report.records) {
+        const failures = blockingResults(record);
+        const skips = skippedResults(record);
+        const attrs = `classname="${xmlEscape(report.definitionId)}" name="${xmlEscape(testcaseName(record))}" time="${(record.durationMs / 1000).toFixed(3)}"`;
+        if (failures.length === 0 && skips.length === 0) {
+            lines.push(`  <testcase ${attrs} />`);
+            continue;
+        }
+        lines.push(`  <testcase ${attrs}>`);
+        for (const failure of failures) {
+            lines.push(`    <failure message="${xmlEscape(failureMessage(failure))}">${xmlEscape(failureBody(failure))}</failure>`);
+        }
+        for (const skip of skips) {
+            lines.push(`    <skipped message="${xmlEscape(skip.explanation ?? `${skip.name} skipped`)}" />`);
+        }
+        lines.push("  </testcase>");
+    }
+    lines.push("</testsuite>");
+    return `${lines.join("\n")}\n`;
+}
+async function writeTextFileEnsuringDir(path, content) {
+    const dir = dirname(path);
+    if (dir && dir !== ".")
+        await dntShim.Deno.mkdir(dir, { recursive: true });
+    await dntShim.Deno.writeTextFile(path, content);
+}
+function resolveAgentTargetId(target) {
+    return target.startsWith("agent:") ? target.slice("agent:".length) : target;
+}
+function normalizeUsage(response) {
+    return response.usage
+        ? {
+            inputTokens: response.usage.promptTokens,
+            outputTokens: response.usage.completionTokens,
+            totalTokens: response.usage.totalTokens,
+        }
+        : {};
+}
+function normalizeToolCalls(response) {
+    return response.toolCalls.map((toolCall) => ({
+        name: toolCall.name,
+        status: toolCall.status === "error" ? "error" : "ok",
+        ...(toolCall.error ? { error: toolCall.error } : {}),
+    }));
+}
+function createAgentAdapter(agent, options) {
+    return async ({ example }) => {
+        const started = Date.now();
+        const response = await agent.generate({
+            input: normalizeEvalInputForAgent(example.input),
+            ...(options.model ? { model: options.model } : {}),
+            ...(options.maxOutputTokens ? { maxOutputTokens: options.maxOutputTokens } : {}),
+        });
+        return {
+            text: response.text,
+            trace: {
+                events: response.messages,
+                toolCalls: normalizeToolCalls(response),
+            },
+            usage: normalizeUsage(response),
+            durationMs: Date.now() - started,
+            completed: response.status === "completed",
+            ...(response.status === "error" ? { error: response.text } : {}),
+        };
+    };
+}
+function listEvals(evals, projectDir) {
+    return evals.map((item) => ({
+        id: item.id,
+        name: item.name,
+        target: item.definition.target,
+        source: {
+            filePath: displaySourcePath(item.filePath, projectDir),
+            exportName: item.exportName,
+        },
+    }));
+}
+function printReport(report) {
+    cliLogger.info(`Eval ${report.definitionId}`);
+    cliLogger.info(`Target: ${report.target}`);
+    cliLogger.info(`Result: ${report.summary.passed}/${report.summary.records} passed (${Math.round(report.summary.passRate * 100)}%)`);
+    for (const metric of report.summary.metrics) {
+        cliLogger.info(`  ${metric.name}: ${metric.passed}/${metric.passed + metric.failed} passed (${Math.round(metric.passRate * 100)}%)`);
+    }
+}
+async function outputEvalNotFound(id, evals) {
+    if (isJsonMode()) {
+        await outputJson(createErrorEnvelope("eval", {
+            code: "NOT_FOUND",
+            slug: "eval-not-found",
+            message: `Eval "${id}" not found`,
+            context: { available: evals.map((item) => item.id) },
+        }));
+    }
+    else {
+        cliLogger.error(`Eval "${id}" not found.`);
+        if (evals.length > 0) {
+            cliLogger.info("Available evals:");
+            for (const item of evals)
+                cliLogger.info(`  - ${item.id}`);
+        }
+        else {
+            cliLogger.info("No evals found. Create an eval file in evals/.");
+        }
+    }
+    exitProcess(1);
+}
+async function outputAgentNotFound(agentId) {
+    if (isJsonMode()) {
+        await outputJson(createErrorEnvelope("eval", {
+            code: "NOT_FOUND",
+            slug: "eval-agent-not-found",
+            message: `Agent "${agentId}" not found`,
+        }));
+    }
+    else {
+        cliLogger.error(`Agent "${agentId}" not found for eval target.`);
+    }
+    exitProcess(1);
+}
+export async function evalCommand(options) {
+    const projectDir = dntShim.Deno.cwd();
+    await withProjectSourceContext(projectDir, async ({ adapter, config }) => {
+        const evalDiscovery = await discoverEvals({ projectDir, adapter, config });
+        if (options.debug) {
+            for (const error of evalDiscovery.errors) {
+                cliLogger.warn(`Eval discovery warning: ${error.filePath}: ${error.error}`);
+            }
+        }
+        if (options.list) {
+            const evals = listEvals(evalDiscovery.evals, projectDir);
+            if (isJsonMode()) {
+                await outputJson(createSuccessEnvelope("eval", { evals, errors: evalDiscovery.errors }));
+                return;
+            }
+            if (evals.length === 0) {
+                cliLogger.info("No evals found.");
+                return;
+            }
+            cliLogger.info("Evals:");
+            for (const item of evals) {
+                cliLogger.info(`  - ${item.id} (${item.target})`);
+            }
+            return;
+        }
+        if (!options.id) {
+            if (isJsonMode()) {
+                await outputJson(createErrorEnvelope("eval", {
+                    code: "USAGE_ERROR",
+                    slug: "eval-id-required",
+                    message: "Eval id is required",
+                }));
+            }
+            else {
+                cliLogger.error("Eval id is required. Usage: veryfront eval <eval-id>");
+            }
+            exitProcess(2);
+            return;
+        }
+        const evalId = normalizeEvalCliId(options.id);
+        const evalItem = evalDiscovery.evals.find((item) => item.id === evalId);
+        if (!evalItem) {
+            await outputEvalNotFound(evalId, evalDiscovery.evals);
+            return;
+        }
+        const discoveryConfig = createProjectDiscoveryConfig({
+            projectDir,
+            config,
+            fsAdapter: adapter.fs,
+            verbose: options.debug,
+        });
+        const projectDiscovery = await discoverAll(discoveryConfig);
+        const agentId = resolveAgentTargetId(evalItem.definition.target);
+        const agent = projectDiscovery.agents.get(agentId);
+        if (!agent) {
+            await outputAgentNotFound(agentId);
+            return;
+        }
+        const report = await runEval(evalItem.definition, {
+            baseDir: options.datasetBase ?? projectDir,
+            adapters: {
+                agent: createAgentAdapter(agent, options),
+            },
+        });
+        if (options.report) {
+            await writeTextFileEnsuringDir(options.report, JSON.stringify(report, null, 2));
+        }
+        if (options.junit) {
+            await writeTextFileEnsuringDir(options.junit, createJunitXml(report));
+        }
+        if (isJsonMode()) {
+            await outputJson(createSuccessEnvelope("eval", {
+                report,
+                summary: summarizeReportForCli(report),
+            }));
+        }
+        else {
+            printReport(report);
+            if (options.report)
+                cliLogger.info(`Report: ${options.report}`);
+            if (options.junit)
+                cliLogger.info(`JUnit: ${options.junit}`);
+        }
+        exitProcess(report.summary.failed === 0 ? 0 : 1);
+    });
+}

package/esm/cli/commands/eval/handler.d.ts

@@ -0,0 +1,26 @@
+import type { InferSchema } from "../../../src/extensions/schema/index.js";
+import type { ParsedArgs } from "../../shared/types.js";
+declare const getEvalArgsSchema: () => import("../../../src/internal-agents/schema.js").Schema<import("../../../src/extensions/schema/schema-validator.js").InferShape<{
+    id: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
+    list: import("../../../src/internal-agents/schema.js").Schema<boolean>;
+    datasetBase: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
+    report: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
+    junit: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
+    debug: import("../../../src/internal-agents/schema.js").Schema<boolean>;
+    model: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
+    maxOutputTokens: import("../../../src/internal-agents/schema.js").Schema<number | undefined>;
+}>>;
+export type EvalArgs = InferSchema<ReturnType<typeof getEvalArgsSchema>>;
+export declare const parseEvalArgs: (args: ParsedArgs) => import("../../shared/args.js").SafeParseResult<import("../../../src/extensions/schema/schema-validator.js").InferShape<{
+    id: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
+    list: import("../../../src/internal-agents/schema.js").Schema<boolean>;
+    datasetBase: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
+    report: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
+    junit: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
+    debug: import("../../../src/internal-agents/schema.js").Schema<boolean>;
+    model: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
+    maxOutputTokens: import("../../../src/internal-agents/schema.js").Schema<number | undefined>;
+}>>;
+export declare function handleEvalCommand(args: ParsedArgs): Promise<void>;
+export {};
+//# sourceMappingURL=handler.d.ts.map

package/esm/cli/commands/eval/handler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/handler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yCAAyC,CAAC;AAE3E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAExD,QAAA,MAAM,iBAAiB;;;;;;;;;GAWtB,CAAC;AAIF,MAAM,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,iBAAiB,CAAC,CAAC,CAAC;AAEzE,eAAO,MAAM,aAAa;;;;;;;;;GASxB,CAAC;AAEH,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAIvE"}

package/esm/cli/commands/eval/handler.js

@@ -0,0 +1,28 @@
+import { defineSchema, lazySchema } from "../../../src/schemas/index.js";
+import { createArgParser, parseArgsOrThrow } from "../../shared/args.js";
+const getEvalArgsSchema = defineSchema((v) => v.object({
+    id: v.string().optional(),
+    list: v.boolean().default(false),
+    datasetBase: v.string().optional(),
+    report: v.string().optional(),
+    junit: v.string().optional(),
+    debug: v.boolean().default(false),
+    model: v.string().optional(),
+    maxOutputTokens: v.number().int().positive().optional(),
+}));
+const EvalArgsSchema = lazySchema(getEvalArgsSchema);
+export const parseEvalArgs = createArgParser(EvalArgsSchema, {
+    id: { keys: ["id"], type: "string", positional: 0 },
+    list: { keys: ["list", "l"], type: "boolean" },
+    datasetBase: { keys: ["dataset-base"], type: "string" },
+    report: { keys: ["report"], type: "string" },
+    junit: { keys: ["junit"], type: "string" },
+    debug: { keys: ["debug"], type: "boolean" },
+    model: { keys: ["model"], type: "string" },
+    maxOutputTokens: { keys: ["max-output-tokens"], type: "number" },
+});
+export async function handleEvalCommand(args) {
+    const opts = parseArgsOrThrow(parseEvalArgs, "eval", args);
+    const { evalCommand } = await import("./command.js");
+    await evalCommand(opts);
+}

package/esm/cli/help/command-definitions.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"command-definitions.d.ts","sourceRoot":"","sources":["../../../src/cli/help/command-definitions.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AA2ClD;;;GAGG;AACH,eAAO,MAAM,QAAQ,EAAE,eAyCtB,CAAC"}
+{"version":3,"file":"command-definitions.d.ts","sourceRoot":"","sources":["../../../src/cli/help/command-definitions.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AA4ClD;;;GAGG;AACH,eAAO,MAAM,QAAQ,EAAE,eA0CtB,CAAC"}

package/esm/cli/help/command-definitions.js

@@ -34,6 +34,7 @@ import { mcpHelp } from "../commands/mcp/command-help.js";
 import { issuesHelp } from "../commands/issues/command-help.js";
 import { startHelp } from "../commands/start/command-help.js";
 import { taskHelp } from "../commands/task/command-help.js";
+import { evalHelp } from "../commands/eval/command-help.js";
 import { workerHelp } from "../commands/worker/command-help.js";
 import { workflowHelp } from "../commands/workflow/command-help.js";
 import { schemaHelp } from "../commands/schema/command-help.js";
@@ -79,6 +80,7 @@ export const COMMANDS = {
     issues: issuesHelp,
     start: startHelp,
     task: taskHelp,
+    eval: evalHelp,
     worker: workerHelp,
     workflow: workflowHelp,
     schema: schemaHelp,

package/esm/cli/router.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../../src/cli/router.ts"],"names":[],"mappings":"AA4DA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAgHpD;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAyHlE"}
+{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../../src/cli/router.ts"],"names":[],"mappings":"AA6DA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAiHpD;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAyHlE"}

package/esm/cli/router.js

@@ -33,6 +33,7 @@ import { handleStudioCommand } from "./commands/studio/handler.js";
 import { handleStylesCommand } from "./commands/styles/handler.js";
 import { handleUpCommand } from "./commands/up/index.js";
 import { handleTaskCommand } from "./commands/task/handler.js";
+import { handleEvalCommand } from "./commands/eval/handler.js";
 import { handleWorkflowCommand } from "./commands/workflow/handler.js";
 import { handleWorkerCommand } from "./commands/worker/handler.js";
 import { handleSchemaCommand } from "./commands/schema/handler.js";
@@ -114,6 +115,7 @@ const commands = {
     "issues": handleIssuesCommand,
     "start": handleStartCommand,
     "task": handleTaskCommand,
+    "eval": handleEvalCommand,
     "workflow": handleWorkflowCommand,
     "worker": handleWorkerCommand,
     "schema": handleSchemaCommand,

package/esm/deno.d.ts

@@ -22,7 +22,7 @@ declare namespace _default {
         "./mdx": string;
         "./agent": string;
         "./agent/identity": string;
-        "./agent/testing": string;
+        "./eval": string;
         "./channels/control-plane": string;
         "./channels/invoke": string;
         "./tool": string;
@@ -89,9 +89,9 @@ declare namespace _default {
         "veryfront/sandbox": string;
         "veryfront/embedding": string;
         "veryfront/runs": string;
+        "veryfront/eval": string;
         "veryfront/agent/identity": string;
         "veryfront/agent/react": string;
-        "veryfront/agent/testing": string;
         "veryfront/agent/middleware": string;
         "veryfront/workflow/react": string;
         "veryfront/workflow/executor": string;
@@ -168,6 +168,7 @@ declare namespace _default {
         "#veryfront/discovery": string;
         "#veryfront/data": string;
         "#veryfront/errors": string;
+        "#veryfront/eval": string;
         "#veryfront/html": string;
         "#veryfront/mcp": string;
         "#veryfront/middleware": string;

package/esm/deno.js

@@ -1,6 +1,6 @@
 export default {
     "name": "veryfront",
-    "version": "0.1.870",
+    "version": "0.1.872",
     "license": "Apache-2.0",
     "nodeModulesDir": "auto",
     "minimumDependencyAge": {
@@ -77,7 +77,7 @@ export default {
         "./mdx": "./src/mdx/index.ts",
         "./agent": "./src/agent/index.ts",
         "./agent/identity": "./src/agent/identity-contracts.ts",
-        "./agent/testing": "./src/agent/testing/index.ts",
+        "./eval": "./src/eval/index.ts",
         "./channels/control-plane": "./src/channels/control-plane.ts",
         "./channels/invoke": "./src/channels/invoke.ts",
         "./tool": "./src/tool/index.ts",
@@ -144,9 +144,9 @@ export default {
         "veryfront/sandbox": "./src/sandbox/index.ts",
         "veryfront/embedding": "./src/embedding/index.ts",
         "veryfront/runs": "./src/runs/index.ts",
+        "veryfront/eval": "./src/eval/index.ts",
         "veryfront/agent/identity": "./src/agent/identity-contracts.ts",
         "veryfront/agent/react": "./src/agent/react/index.ts",
-        "veryfront/agent/testing": "./src/agent/testing/index.ts",
         "veryfront/agent/middleware": "./src/agent/middleware/index.ts",
         "veryfront/workflow/react": "./src/workflow/react/index.ts",
         "veryfront/workflow/executor": "./src/workflow/executor/index.ts",
@@ -223,6 +223,7 @@ export default {
         "#veryfront/discovery": "./src/discovery/index.ts",
         "#veryfront/data": "./src/data/index.ts",
         "#veryfront/errors": "./src/errors/index.ts",
+        "#veryfront/eval": "./src/eval/index.ts",
         "#veryfront/html": "./src/html/index.ts",
         "#veryfront/mcp": "./src/mcp/index.ts",
         "#veryfront/middleware": "./src/middleware/index.ts",
@@ -373,7 +374,7 @@ export default {
         "lint": "DENO_NO_PACKAGE_JSON=1 deno lint src/ cli/ react/ && deno lint --config=scripts/test.deno.json scripts/test/ scripts/build/npm-package-metadata.test.ts",
         "fmt": "deno fmt src/ cli/ react/ && deno fmt --config=scripts/test.deno.json scripts/test/",
         "fmt:check": "deno fmt --check src/ cli/ react/ && deno fmt --check --config=scripts/test.deno.json scripts/test/",
-        "typecheck": "deno task generate:manifests:check && deno check src/index.ts cli/main.ts src/server/index.ts src/routing/api/index.ts src/rendering/index.ts src/platform/index.ts src/platform/adapters/index.ts src/build/index.ts src/build/production-build/index.ts src/transforms/index.ts src/config/index.ts src/utils/index.ts src/data/index.ts src/security/index.ts src/middleware/index.ts src/server/handlers/dev/index.ts src/server/handlers/request/api/index.ts src/rendering/cache/index.ts src/rendering/cache/stores/index.ts src/rendering/rsc/actions/index.ts src/html/index.ts src/modules/index.ts src/proxy/main.ts src/chat/index.ts src/markdown/index.ts src/mdx/index.ts src/fs/index.ts src/oauth/index.ts src/agent/index.ts src/agent/service/route-export.check.ts src/tool/index.ts src/workflow/index.ts src/work/index.ts src/prompt/index.ts src/resource/index.ts src/runs/index.ts src/mcp/index.ts src/provider/index.ts",
+        "typecheck": "deno task generate:manifests:check && deno check src/index.ts cli/main.ts src/server/index.ts src/routing/api/index.ts src/rendering/index.ts src/platform/index.ts src/platform/adapters/index.ts src/build/index.ts src/build/production-build/index.ts src/transforms/index.ts src/config/index.ts src/utils/index.ts src/data/index.ts src/security/index.ts src/middleware/index.ts src/server/handlers/dev/index.ts src/server/handlers/request/api/index.ts src/rendering/cache/index.ts src/rendering/cache/stores/index.ts src/rendering/rsc/actions/index.ts src/html/index.ts src/modules/index.ts src/proxy/main.ts src/chat/index.ts src/markdown/index.ts src/mdx/index.ts src/fs/index.ts src/oauth/index.ts src/agent/index.ts src/agent/service/route-export.check.ts src/eval/index.ts src/tool/index.ts src/workflow/index.ts src/work/index.ts src/prompt/index.ts src/resource/index.ts src/runs/index.ts src/mcp/index.ts src/provider/index.ts",
         "verify": "deno task generate:manifests:check && deno task fmt:check && deno task lint && deno task lint:style && deno task lint:cli-boundary && deno task lint:wildcard-exports && deno task lint:barrel-jsdoc && deno task lint:ban-test-only && deno task lint:sanitizer-baseline && deno task lint:skipped-tests && deno task lint:ban-zod && deno task lint:core-deps && deno task lint:dependency-boundaries && deno task lint:extension-contracts && deno task lint:extension-capabilities && deno task docs:validate && deno task typecheck && deno task test && deno task test:e2e:binary",
         "verify:quick": "deno task generate:manifests:check && deno task fmt:check && deno task lint && deno task lint:style && deno task lint:cli-boundary && deno task lint:wildcard-exports && deno task lint:barrel-jsdoc && deno task lint:ban-test-only && deno task lint:sanitizer-baseline && deno task lint:skipped-tests && deno task lint:ban-zod && deno task lint:core-deps && deno task lint:dependency-boundaries && deno task lint:extension-contracts && deno task lint:extension-capabilities && deno task docs:validate && deno task typecheck",
         "docs": "deno run --allow-read --allow-write --allow-run --allow-env scripts/docs/generate-api-reference.ts",

package/esm/src/agent/runtime/client-profile.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client-profile.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/runtime/client-profile.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kCAAkC,CAAC;AAEpE,8CAA8C;AAC9C,eAAO,MAAM,0BAA0B,gEAEtC,CAAC;AAEF,oDAAoD;AACpD,eAAO,MAAM,gCAAgC,gEAO5C,CAAC;AAEF,iDAAiD;AACjD,eAAO,MAAM,6BAA6B;;;;;GAOzC,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,uBAAuB,0DAAyC,CAAC;AAC9E;;GAEG;AACH,eAAO,MAAM,6BAA6B,0DAA+C,CAAC;AAC1F;;GAEG;AACH,eAAO,MAAM,0BAA0B;;;;;GAA4C,CAAC;AAEpF,mDAAmD;AACnD,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC;AAC3F,yDAAyD;AACzD,MAAM,MAAM,uBAAuB,GAAG,WAAW,CAC/C,UAAU,CAAC,OAAO,gCAAgC,CAAC,CACpD,CAAC;AACF,sDAAsD;AACtD,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,6BAA6B,CAAC,CAAC,CAAC;AA0CjG,uCAAuC;AACvC,wBAAgB,2BAA2B,CACzC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GAClD,oBAAoB,GAAG,IAAI,CA4B7B;AAED,uCAAuC;AACvC,wBAAgB,qBAAqB,CACnC,aAAa,EAAE,oBAAoB,GAAG,IAAI,GAAG,SAAS,GACrD,OAAO,CAUT"}
+{"version":3,"file":"client-profile.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/runtime/client-profile.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kCAAkC,CAAC;AAEpE,8CAA8C;AAC9C,eAAO,MAAM,0BAA0B,gEAEtC,CAAC;AAEF,oDAAoD;AACpD,eAAO,MAAM,gCAAgC,gEAS5C,CAAC;AAEF,iDAAiD;AACjD,eAAO,MAAM,6BAA6B;;;;;GAOzC,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,uBAAuB,0DAAyC,CAAC;AAC9E;;GAEG;AACH,eAAO,MAAM,6BAA6B,0DAA+C,CAAC;AAC1F;;GAEG;AACH,eAAO,MAAM,0BAA0B;;;;;GAA4C,CAAC;AAEpF,mDAAmD;AACnD,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC;AAC3F,yDAAyD;AACzD,MAAM,MAAM,uBAAuB,GAAG,WAAW,CAC/C,UAAU,CAAC,OAAO,gCAAgC,CAAC,CACpD,CAAC;AACF,sDAAsD;AACtD,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,6BAA6B,CAAC,CAAC,CAAC;AAiDjG,uCAAuC;AACvC,wBAAgB,2BAA2B,CACzC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GAClD,oBAAoB,GAAG,IAAI,CA4B7B;AAED,uCAAuC;AACvC,wBAAgB,qBAAqB,CACnC,aAAa,EAAE,oBAAoB,GAAG,IAAI,GAAG,SAAS,GACrD,OAAO,CAUT"}

package/esm/src/agent/runtime/client-profile.js

@@ -7,6 +7,8 @@ export const getRuntimeClientCapabilitySchema = defineSchema((v) => v.enum([
     "form_input",
     "media_display",
     "project_switching",
+    "project.evals.read",
+    "project.evals.write",
 ]));
 /** Zod schema for get runtime client profile. */
 export const getRuntimeClientProfileSchema = defineSchema((v) => v.object({
@@ -44,7 +46,14 @@ const getClientEnvelopeSchema = defineSchema((v) => v.object({
 const FIRST_PARTY_CLIENTS = {
     "veryfront-studio": {
         type: "web",
-        capabilities: ["ui_panels", "form_input", "media_display", "project_switching"],
+        capabilities: [
+            "ui_panels",
+            "form_input",
+            "media_display",
+            "project_switching",
+            "project.evals.read",
+            "project.evals.write",
+        ],
     },
     "veryfront-cli": {
         type: "cli",

package/esm/src/config/schemas/config.schema.d.ts

@@ -313,6 +313,12 @@ export declare const getVeryfrontConfigSchema: () => import("../../internal-agen
                 paths: import("../../internal-agents/schema.js").Schema<string[] | undefined>;
             }>> | undefined>;
         }>> | undefined>;
+        evals: import("../../internal-agents/schema.js").Schema<Partial<import("../../extensions/schema/schema-validator.js").InferShape<{
+            discovery: import("../../internal-agents/schema.js").Schema<Partial<import("../../extensions/schema/schema-validator.js").InferShape<{
+                enabled: import("../../internal-agents/schema.js").Schema<boolean | undefined>;
+                paths: import("../../internal-agents/schema.js").Schema<string[] | undefined>;
+            }>> | undefined>;
+        }>> | undefined>;
         mcp: import("../../internal-agents/schema.js").Schema<Partial<import("../../extensions/schema/schema-validator.js").InferShape<{
             enabled: import("../../internal-agents/schema.js").Schema<boolean | undefined>;
             port: import("../../internal-agents/schema.js").Schema<number | undefined>;
@@ -717,6 +723,12 @@ export declare const veryfrontConfigSchema: import("../../internal-agents/schema
                 paths: import("../../internal-agents/schema.js").Schema<string[] | undefined>;
             }>> | undefined>;
         }>> | undefined>;
+        evals: import("../../internal-agents/schema.js").Schema<Partial<import("../../extensions/schema/schema-validator.js").InferShape<{
+            discovery: import("../../internal-agents/schema.js").Schema<Partial<import("../../extensions/schema/schema-validator.js").InferShape<{
+                enabled: import("../../internal-agents/schema.js").Schema<boolean | undefined>;
+                paths: import("../../internal-agents/schema.js").Schema<string[] | undefined>;
+            }>> | undefined>;
+        }>> | undefined>;
         mcp: import("../../internal-agents/schema.js").Schema<Partial<import("../../extensions/schema/schema-validator.js").InferShape<{
             enabled: import("../../internal-agents/schema.js").Schema<boolean | undefined>;
             port: import("../../internal-agents/schema.js").Schema<number | undefined>;

package/esm/src/config/schemas/config.schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.schema.d.ts","sourceRoot":"","sources":["../../../../src/src/config/schemas/config.schema.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,kCAAkC,CAAC;AAiChF,eAAO,MAAM,wBAAwB;;;;;QAQ3B,2GAA2G;;;;;;;;;;;;;;;IAuB/G,wFAAwF;;IAExF,0FAA0F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAyCtF;;;;;;;;;;;;;;;;;WAiBG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAgDH;;;;;;;;;;WAUG;;;;;;;;;;QAaH;;;;;WAKG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YA4HC,kFAAkF;;YAElF,oFAAoF;;YAEpF,+EAA+E;;YAE/E,mEAAmE;;;;;;;;;;;;;;;;;YA8BnE,mCAAmC;;YAEnC,8CAA8C;;YAE9C,sBAAsB;;YAEtB,mDAAmD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAiJvD,yDAAyD;;QAEzD,iDAAiD;;;YAI7C,iEAAiE;;;;;;;IAgBzE,uCAAuC;;QAGnC,2CAA2C;;;;QAO3C,6DAA6D;;QAE7D,gGAAgG;;QAGhG,iEAAiE;;;;QAOjE,sEAAsE;;;IAK1E,kEAAkE;;QAM1D,yEAAyE;;QAEzE,kDAAkD;;QAElD;;sDAE8C;;;IAOtD;;;;;;;;;OASG;;IAEH,0CAA0C;;QAGtC,8CAA8C;;QAE9C,gEAAgE;;QAEhE,yCAAyC;;QAEzC,qCAAqC;;QAErC,sBAAsB;;QAEtB,gCAAgC;;YAG5B,qDAAqD;;YAErD,qDAAqD;;YAErD,oDAAoD;;;QAKxD,oCAAoC;;YAGhC,4EAA4E;;YAE5E,8DAA8D;;YAE9D,8EAA8E;;;;IAU3F,CAAC;AACF,eAAO,MAAM,qBAAqB;;;;;QAhlBxB,2GAA2G;;;;;;;;;;;;;;;IAuB/G,wFAAwF;;IAExF,0FAA0F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAyCtF;;;;;;;;;;;;;;;;;WAiBG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAgDH;;;;;;;;;;WAUG;;;;;;;;;;QAaH;;;;;WAKG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YA4HC,kFAAkF;;YAElF,oFAAoF;;YAEpF,+EAA+E;;YAE/E,mEAAmE;;;;;;;;;;;;;;;;;YA8BnE,mCAAmC;;YAEnC,8CAA8C;;YAE9C,sBAAsB;;YAEtB,mDAAmD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAiJvD,yDAAyD;;QAEzD,iDAAiD;;;YAI7C,iEAAiE;;;;;;;IAgBzE,uCAAuC;;QAGnC,2CAA2C;;;;QAO3C,6DAA6D;;QAE7D,gGAAgG;;QAGhG,iEAAiE;;;;QAOjE,sEAAsE;;;IAK1E,kEAAkE;;QAM1D,yEAAyE;;QAEzE,kDAAkD;;QAElD;;sDAE8C;;;IAOtD;;;;;;;;;OASG;;IAEH,0CAA0C;;QAGtC,8CAA8C;;QAE9C,gEAAgE;;QAEhE,yCAAyC;;QAEzC,qCAAqC;;QAErC,sBAAsB;;QAEtB,gCAAgC;;YAG5B,qDAAqD;;YAErD,qDAAqD;;YAErD,oDAAoD;;;QAKxD,oCAAoC;;YAGhC,4EAA4E;;YAE5E,8DAA8D;;YAE9D,8EAA8E;;;;IAWnB,CAAC;AAG1E,MAAM,MAAM,eAAe,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,wBAAwB,CAAC,CAAC,CAAC;AACvF,MAAM,MAAM,oBAAoB,GAAG,UAAU,CAAC,UAAU,CAAC,OAAO,wBAAwB,CAAC,CAAC,CAAC;AAG3F,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,eAAe,CA0BvE;AAqCD,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,EAAE,CAEhF"}
+{"version":3,"file":"config.schema.d.ts","sourceRoot":"","sources":["../../../../src/src/config/schemas/config.schema.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,kCAAkC,CAAC;AAiChF,eAAO,MAAM,wBAAwB;;;;;QAQ3B,2GAA2G;;;;;;;;;;;;;;;IAuB/G,wFAAwF;;IAExF,0FAA0F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAyCtF;;;;;;;;;;;;;;;;;WAiBG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAgDH;;;;;;;;;;WAUG;;;;;;;;;;QAaH;;;;;WAKG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YA4HC,kFAAkF;;YAElF,oFAAoF;;YAEpF,+EAA+E;;YAE/E,mEAAmE;;;;;;;;;;;;;;;;;YA8BnE,mCAAmC;;YAEnC,8CAA8C;;YAE9C,sBAAsB;;YAEtB,mDAAmD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QA6JvD,yDAAyD;;QAEzD,iDAAiD;;;YAI7C,iEAAiE;;;;;;;IAgBzE,uCAAuC;;QAGnC,2CAA2C;;;;QAO3C,6DAA6D;;QAE7D,gGAAgG;;QAGhG,iEAAiE;;;;QAOjE,sEAAsE;;;IAK1E,kEAAkE;;QAM1D,yEAAyE;;QAEzE,kDAAkD;;QAElD;;sDAE8C;;;IAOtD;;;;;;;;;OASG;;IAEH,0CAA0C;;QAGtC,8CAA8C;;QAE9C,gEAAgE;;QAEhE,yCAAyC;;QAEzC,qCAAqC;;QAErC,sBAAsB;;QAEtB,gCAAgC;;YAG5B,qDAAqD;;YAErD,qDAAqD;;YAErD,oDAAoD;;;QAKxD,oCAAoC;;YAGhC,4EAA4E;;YAE5E,8DAA8D;;YAE9D,8EAA8E;;;;IAU3F,CAAC;AACF,eAAO,MAAM,qBAAqB;;;;;QA5lBxB,2GAA2G;;;;;;;;;;;;;;;IAuB/G,wFAAwF;;IAExF,0FAA0F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAyCtF;;;;;;;;;;;;;;;;;WAiBG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAgDH;;;;;;;;;;WAUG;;;;;;;;;;QAaH;;;;;WAKG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YA4HC,kFAAkF;;YAElF,oFAAoF;;YAEpF,+EAA+E;;YAE/E,mEAAmE;;;;;;;;;;;;;;;;;YA8BnE,mCAAmC;;YAEnC,8CAA8C;;YAE9C,sBAAsB;;YAEtB,mDAAmD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QA6JvD,yDAAyD;;QAEzD,iDAAiD;;;YAI7C,iEAAiE;;;;;;;IAgBzE,uCAAuC;;QAGnC,2CAA2C;;;;QAO3C,6DAA6D;;QAE7D,gGAAgG;;QAGhG,iEAAiE;;;;QAOjE,sEAAsE;;;IAK1E,kEAAkE;;QAM1D,yEAAyE;;QAEzE,kDAAkD;;QAElD;;sDAE8C;;;IAOtD;;;;;;;;;OASG;;IAEH,0CAA0C;;QAGtC,8CAA8C;;QAE9C,gEAAgE;;QAEhE,yCAAyC;;QAEzC,qCAAqC;;QAErC,sBAAsB;;QAEtB,gCAAgC;;YAG5B,qDAAqD;;YAErD,qDAAqD;;YAErD,oDAAoD;;;QAKxD,oCAAoC;;YAGhC,4EAA4E;;YAE5E,8DAA8D;;YAE9D,8EAA8E;;;;IAWnB,CAAC;AAG1E,MAAM,MAAM,eAAe,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,wBAAwB,CAAC,CAAC,CAAC;AACvF,MAAM,MAAM,oBAAoB,GAAG,UAAU,CAAC,UAAU,CAAC,OAAO,wBAAwB,CAAC,CAAC,CAAC;AAG3F,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,eAAe,CA0BvE;AAqCD,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,EAAE,CAEhF"}

package/esm/src/config/schemas/config.schema.js

@@ -479,6 +479,18 @@ export const getVeryfrontConfigSchema = defineSchema((v) => v
         })
             .partial()
             .optional(),
+        evals: v
+            .object({
+            discovery: v
+                .object({
+                enabled: v.boolean().optional(),
+                paths: v.array(v.string()).optional(),
+            })
+                .partial()
+                .optional(),
+        })
+            .partial()
+            .optional(),
         mcp: v
             .object({
             enabled: v.boolean().optional(),