veryfront 0.1.1057 → 0.1.1060

package/esm/cli/commands/eval/command.d.ts

@@ -1,6 +1,7 @@
 import type { AgentResponse } from "../../../src/agent/index.js";
 import type { VeryfrontConfig } from "../../../src/config/index.js";
-import type { DiscoveredEval, EvalModelComparison, EvalModelComparisonOptions, EvalReport, EvalReportComparison, EvalReportExportConfig, EvalToolCall } from "../../../src/eval/index.js";
+import { type Tool, type ToolExecutionContext } from "../../../src/tool/index.js";
+import type { DiscoveredEval, EvalModelComparison, EvalModelComparisonOptions, EvalReport, EvalReportComparison, EvalReportExportConfig, EvalToolAdapterContext, EvalToolCall } from "../../../src/eval/index.js";
 import type { EvalArgs } from "./handler.js";
 export interface EvalOptions extends EvalArgs {
 }
@@ -74,12 +75,20 @@ export declare function createEvalMarkdownReport(report: EvalReport, baseline?:
 export declare function createJunitXml(report: EvalReport): string;
 export declare function writeEvalArtifacts(report: EvalReport, paths: EvalArtifactPaths, baseline?: EvalReportComparison): Promise<void>;
 export declare function createEvalExitCode(report: EvalReport, baseline?: EvalReportComparison): 0 | 1;
+export declare function resolveToolTargetId(target: string): string;
 type EvalRuntimeAuthConfig = Pick<VeryfrontConfig, "projectSlug" | "fs"> & {
     projectSlug?: string;
 };
 export declare function hydrateEvalRuntimeAuth(projectDir: string, config: EvalRuntimeAuthConfig | null | undefined): Promise<import("../../shared/runtime-auth.js").RuntimeAuthContext>;
 export declare function normalizeUsage(response: AgentResponse): {};
 export declare function normalizeToolCalls(response: AgentResponse): EvalToolCall[];
+export declare function createToolAdapter(tool: Tool, baseContext?: ToolExecutionContext): (context: EvalToolAdapterContext) => Promise<{
+    error?: string | undefined;
+    output: any;
+    toolCallId: string;
+    durationMs: number;
+    completed: boolean;
+}>;
 type EvalModelComparisonConfig = {
     baselineModel: string;
     candidateModels: string[];

package/esm/cli/commands/eval/command.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EAEpB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AA2BpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,+BAA+B,GAAG;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAOF,KAAK,6BAA6B,GAAG;IACnC,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AAoDnF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhF;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAO5E;AAMD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CASxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAgBrB;AA8GD,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,+BAA+B,GAC5C,UAAU,CAiBZ;AAWD,wBAAsB,2BAA2B,CAC/C,cAAc,EAAE,MAAM,EACtB,OAAO,GAAE,6BAAkC,GAC1C,OAAO,CAAC,+BAA+B,GAAG,SAAS,CAAC,CAgDtD;AAED,wBAAsB,8BAA8B,CAClD,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,GACnC,OAAO,CAAC,UAAU,CAAC,CAerB;AAED,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,UAAU,CAAC,CAGrB;AA8FD,wBAAsB,6BAA6B,CACjD,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,yBAAyB,CAAC,CA0CpC;AAED,wBAAsB,uCAAuC,CAC3D,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,iCAAiC,GAAG,SAAS,CAAC,CAKxD;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,EACrB,MAAM,GAAE,yBAA8B,GACrC,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAE9E;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAkED,uEAAuE;AACvE,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,MAAM,CAkFR;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAQf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,CAAC,GAAG,CAAC,CAEP;AAMD,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,MA0DrD;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AA+FD,KAAK,yBAAyB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,KAAK,iCAAiC,GAAG;IACvC,MAAM,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AA8LF,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CA0JrE"}
+{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAgC,KAAK,IAAI,EAAE,KAAK,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAChH,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EAEpB,sBAAsB,EACtB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AA2BpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,+BAA+B,GAAG;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAOF,KAAK,6BAA6B,GAAG;IACnC,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AAoDnF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhF;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAO5E;AAMD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CASxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAgBrB;AA8GD,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,+BAA+B,GAC5C,UAAU,CAiBZ;AAWD,wBAAsB,2BAA2B,CAC/C,cAAc,EAAE,MAAM,EACtB,OAAO,GAAE,6BAAkC,GAC1C,OAAO,CAAC,+BAA+B,GAAG,SAAS,CAAC,CAgDtD;AAED,wBAAsB,8BAA8B,CAClD,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,GACnC,OAAO,CAAC,UAAU,CAAC,CAerB;AAED,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,UAAU,CAAC,CAGrB;AA8FD,wBAAsB,6BAA6B,CACjD,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,yBAAyB,CAAC,CA0CpC;AAED,wBAAsB,uCAAuC,CAC3D,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,iCAAiC,GAAG,SAAS,CAAC,CAKxD;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,EACrB,MAAM,GAAE,yBAA8B,GACrC,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAE9E;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAkED,uEAAuE;AACvE,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,MAAM,CAkFR;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAQf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,CAAC,GAAG,CAAC,CAEP;AAMD,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAaD,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,MA0DrD;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AAqCD,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,GAAE,oBAAyB,IACpE,SAAS,sBAAsB;;;;;;GAiB9C;AAyED,KAAK,yBAAyB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,KAAK,iCAAiC,GAAG;IACvC,MAAM,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AA2MF,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAkLrE"}

package/esm/cli/commands/eval/command.js

@@ -3,6 +3,7 @@
  */
 import * as dntShim from "../../../_dnt.shims.js";
 import { dirname, isAbsolute, join, relative, resolve } from "../../../deps/jsr.io/@std/path/1.1.4/mod.js";
+import { isErroredToolExecutionResult } from "../../../src/tool/index.js";
 import { createProjectDiscoveryConfig, discoverAll } from "../../../src/discovery/index.js";
 import { compareEvalModelReports, compareEvalReports, createEvalModelComparisonMarkdown, createEvalRunId, discoverEvals, EVAL_REPORT_SCHEMA_VERSION, exportEvalReport, resolveEvalRunProvenance, runEval, } from "../../../src/eval/index.js";
 import { getCurrentVeryfrontCloudContext, getVeryfrontCloudBootstrap, runWithVeryfrontCloudContextAsync, } from "../../../src/provider/index.js";
@@ -624,6 +625,9 @@ export function createEvalExitCode(report, baseline) {
 function resolveAgentTargetId(target) {
     return target.startsWith("agent:") ? target.slice("agent:".length) : target;
 }
+export function resolveToolTargetId(target) {
+    return target.startsWith("tool:") ? target.slice("tool:".length) : target;
+}
 function resolveEvalRuntimeProjectSlug(config) {
     return config?.projectSlug ?? config?.fs?.veryfront?.projectSlug;
 }
@@ -633,6 +637,14 @@ export async function hydrateEvalRuntimeAuth(projectDir, config) {
         projectSlug: resolveEvalRuntimeProjectSlug(config),
     });
 }
+function createEvalToolExecutionContext(config) {
+    const projectSlug = resolveEvalRuntimeProjectSlug(config);
+    const authToken = dntShim.Deno.env.get("VERYFRONT_API_TOKEN");
+    return {
+        ...(projectSlug ? { projectSlug } : {}),
+        ...(authToken ? { authToken } : {}),
+    };
+}
 export function normalizeUsage(response) {
     return response.usage
         ? {
@@ -705,6 +717,19 @@ export function normalizeToolCalls(response) {
             : {}),
     }));
 }
+function getToolExecutionErrorMessage(output) {
+    if (!isErroredToolExecutionResult(output))
+        return undefined;
+    if (isRecord(output) && typeof output.error === "string")
+        return output.error;
+    if (isRecord(output) && isRecord(output.output) && typeof output.output.error === "string") {
+        return output.output.error;
+    }
+    return "Tool execution returned an error result.";
+}
+function createEvalToolCallId(toolId, context) {
+    return `eval-${toolId}-${context.example.id}-${context.repetition}-${dntShim.crypto.randomUUID()}`;
+}
 function createAgentAdapter(agent, options) {
     return async ({ example }) => {
         const started = Date.now();
@@ -726,6 +751,25 @@ function createAgentAdapter(agent, options) {
         };
     };
 }
+export function createToolAdapter(tool, baseContext = {}) {
+    return async (context) => {
+        const started = Date.now();
+        const toolCallId = createEvalToolCallId(tool.id, context);
+        const output = await tool.execute(context.input, {
+            ...baseContext,
+            runId: context.runId,
+            toolCallId,
+        });
+        const error = getToolExecutionErrorMessage(output);
+        return {
+            output,
+            toolCallId,
+            durationMs: Date.now() - started,
+            completed: !error,
+            ...(error ? { error } : {}),
+        };
+    };
+}
 function listEvals(evals, projectDir) {
     return evals.map((item) => ({
         id: item.id,
@@ -910,6 +954,19 @@ async function outputAgentNotFound(agentId) {
     }
     exitProcess(1);
 }
+async function outputToolNotFound(toolId) {
+    if (isJsonMode()) {
+        await outputJson(createErrorEnvelope("eval", {
+            code: "NOT_FOUND",
+            slug: "eval-tool-not-found",
+            message: `Tool "${toolId}" not found`,
+        }));
+    }
+    else {
+        cliLogger.error(`Tool "${toolId}" not found for eval target.`);
+    }
+    exitProcess(1);
+}
 async function outputEvalUsageError(message) {
     if (isJsonMode()) {
         await outputJson(createErrorEnvelope("eval", {
@@ -977,6 +1034,16 @@ export async function evalCommand(options) {
             await outputEvalUsageError(error instanceof Error ? error.message : String(error));
             return;
         }
+        if (evalItem.definition.targetKind !== "agent") {
+            if (modelComparisonConfig) {
+                await outputEvalUsageError("Model comparison flags are only supported for agent evals.");
+                return;
+            }
+            if (options.model || options.maxOutputTokens) {
+                await outputEvalUsageError("--model and --max-output-tokens are only supported for agent evals.");
+                return;
+            }
+        }
         const discoveryConfig = createProjectDiscoveryConfig({
             projectDir,
             config,
@@ -984,16 +1051,26 @@ export async function evalCommand(options) {
             verbose: options.debug,
         });
         const projectDiscovery = await discoverAll(discoveryConfig);
-        const agentId = resolveAgentTargetId(evalItem.definition.target);
-        const agent = projectDiscovery.agents.get(agentId);
-        if (!agent) {
+        const agentId = evalItem.definition.targetKind === "agent"
+            ? resolveAgentTargetId(evalItem.definition.target)
+            : undefined;
+        const toolId = evalItem.definition.targetKind === "tool"
+            ? resolveToolTargetId(evalItem.definition.target)
+            : undefined;
+        const agent = agentId ? projectDiscovery.agents.get(agentId) : undefined;
+        const tool = toolId ? projectDiscovery.tools.get(toolId) : undefined;
+        if (agentId && !agent) {
             await outputAgentNotFound(agentId);
             return;
         }
+        if (toolId && !tool) {
+            await outputToolNotFound(toolId);
+            return;
+        }
         if (modelComparisonConfig) {
             await runEvalModelComparison({
                 evalItem,
-                agent,
+                agent: agent,
                 options,
                 projectDir,
                 config: modelComparisonConfig.config,
@@ -1014,9 +1091,9 @@ export async function evalCommand(options) {
         const finalizedReport = await runEvalWithGatewayBillingGroup(billingGroupId, () => runEval(evalItem.definition, {
             baseDir: options.datasetBase ?? projectDir,
             runId,
-            adapters: {
-                agent: createAgentAdapter(agent, options),
-            },
+            adapters: evalItem.definition.targetKind === "tool"
+                ? { tool: createToolAdapter(tool, createEvalToolExecutionContext(config)) }
+                : { agent: createAgentAdapter(agent, options) },
             metadata: {
                 provenance,
                 ...(options.model ? { model: options.model } : {}),

package/esm/deno.js

@@ -1,6 +1,6 @@
 export default {
     "name": "veryfront",
-    "version": "0.1.1057",
+    "version": "0.1.1060",
     "license": "Apache-2.0",
     "nodeModulesDir": "auto",
     "minimumDependencyAge": {

package/esm/src/eval/factory.d.ts

@@ -1,6 +1,8 @@
-import type { EvalAgentInput, EvalDefinition } from "./types.js";
-/** Define a V1 eval that targets a Veryfront agent. */
+import type { EvalAgentInput, EvalDefinition, EvalToolInput } from "./types.js";
+/** Define an eval that targets a Veryfront agent. */
 export declare function evalAgent(input: EvalAgentInput): EvalDefinition;
+/** Define an eval that targets a Veryfront tool. */
+export declare function evalTool(input: EvalToolInput): EvalDefinition;
 /** Check whether a value is a normalized eval definition. */
 export declare function isEvalDefinition(value: unknown): value is EvalDefinition;
 //# sourceMappingURL=factory.d.ts.map

package/esm/src/eval/factory.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../../src/src/eval/factory.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAe,cAAc,EAAoB,MAAM,YAAY,CAAC;AAwBhG,uDAAuD;AACvD,wBAAgB,SAAS,CAAC,KAAK,EAAE,cAAc,GAAG,cAAc,CAmB/D;AAED,6DAA6D;AAC7D,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,cAAc,CASxE"}
+{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../../src/src/eval/factory.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,cAAc,EAEd,cAAc,EAGd,aAAa,EACd,MAAM,YAAY,CAAC;AAiDpB,qDAAqD;AACrD,wBAAgB,SAAS,CAAC,KAAK,EAAE,cAAc,GAAG,cAAc,CAE/D;AAED,oDAAoD;AACpD,wBAAgB,QAAQ,CAAC,KAAK,EAAE,aAAa,GAAG,cAAc,CAE7D;AAED,6DAA6D;AAC7D,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,cAAc,CASxE"}

package/esm/src/eval/factory.js

@@ -16,14 +16,13 @@ function normalizeRepetitions(repetitions) {
     }
     return value;
 }
-/** Define a V1 eval that targets a Veryfront agent. */
-export function evalAgent(input) {
+function createEvalDefinition(targetKind, input) {
     if (typeof input.target !== "string" || input.target.trim() === "") {
         throw createEvalValidationError("Eval target must be a non-empty string");
     }
     return {
         kind: "eval",
-        targetKind: "agent",
+        targetKind,
         id: input.id ?? "",
         name: input.name ?? input.id ?? input.target,
         ...(input.description ? { description: input.description } : {}),
@@ -33,16 +32,25 @@ export function evalAgent(input) {
         repetitions: normalizeRepetitions(input.repetitions),
         tags: input.tags ?? [],
         metadata: input.metadata ?? {},
+        ...("input" in input && input.input ? { input: input.input } : {}),
         ...(input.check ? { check: input.check } : {}),
     };
 }
+/** Define an eval that targets a Veryfront agent. */
+export function evalAgent(input) {
+    return createEvalDefinition("agent", input);
+}
+/** Define an eval that targets a Veryfront tool. */
+export function evalTool(input) {
+    return createEvalDefinition("tool", input);
+}
 /** Check whether a value is a normalized eval definition. */
 export function isEvalDefinition(value) {
     if (!value || typeof value !== "object")
         return false;
     const candidate = value;
     return candidate.kind === "eval" &&
-        candidate.targetKind === "agent" &&
+        (candidate.targetKind === "agent" || candidate.targetKind === "tool") &&
         typeof candidate.target === "string" &&
         isEvalDataset(candidate.dataset) &&
         Array.isArray(candidate.metrics) &&

package/esm/src/eval/index.d.ts

@@ -44,7 +44,7 @@
  */
 import "../../_dnt.polyfills.js";
 export { datasets } from "./datasets.js";
-export { evalAgent, isEvalDefinition } from "./factory.js";
+export { evalAgent, evalTool, isEvalDefinition } from "./factory.js";
 export { judges } from "./judges.js";
 export { metrics } from "./metrics.js";
 export { createEvalDatasetMetadata, createEvalReport, EVAL_REPORT_SCHEMA_VERSION, summarizeEvalRecords, } from "./report.js";
@@ -54,9 +54,9 @@ export { createEvalRunProvenance, resolveEvalRunProvenance } from "./provenance.
 export { createEvalRunId } from "./run-id.js";
 export { exportEvalReport, runEval } from "./runner.js";
 export { deriveEvalId, discoverEvals, findEvalById } from "./discovery.js";
-export { createEvalSourceDocument, getEvalEditableFieldSchema, getEvalRunSchema, getEvalSourceDocumentSchema, getEvalSourcePatchSchema, getEvalSourceReferenceSchema, getEvalStudioCapabilitySchema, } from "./studio.js";
+export { createEvalSourceDocument, getEvalEditableFieldSchema, getEvalRunSchema, getEvalSourceDocumentSchema, getEvalSourcePatchSchema, getEvalSourceReferenceSchema, getEvalStudioCapabilitySchema, getEvalTargetKindSchema, } from "./studio.js";
 export type { DiscoveredEval, EvalDiscoveryOptions, EvalDiscoveryResult } from "./discovery.js";
 export type { EvalLlmGroundednessJudgeOptions } from "./judges.js";
-export type { EvalAgentAdapter, EvalAgentAdapterContext, EvalAgentAdapterResult, EvalAgentInput, EvalAnswerGroundednessMetricOptions, EvalBudgetDeltaSummary, EvalCheckContext, EvalCitation, EvalDataset, EvalDatasetLoadContext, EvalDefinition, EvalDurationSummary, EvalExample, EvalExampleInput, EvalExpect, EvalExpectation, EvalFailedExampleSummary, EvalFlakeSummary, EvalGateFailureSummary, EvalKnowledgeCitationMetricOptions, EvalKnowledgeExpectedSource, EvalKnowledgeMrrMetricOptions, EvalKnowledgeRetrievalMetricOptions, EvalMetric, EvalMetricContext, EvalMetricDeltaSummary, EvalMetricFamily, EvalMetricResult, EvalMetricSummary, EvalMetricThreshold, EvalModelCandidateComparison, EvalModelComparison, EvalModelComparisonConstraint, EvalModelComparisonDecision, EvalModelComparisonMetricName, EvalModelComparisonObjective, EvalModelComparisonOptions, EvalModelReportSummary, EvalRecord, EvalReport, EvalReportComparison, EvalReportComparisonPolicy, EvalReportDatasetMetadata, EvalReportExportConfig, EvalReportMetadata, EvalReportSummary, EvalRetrievedContext, EvalRunProvenance, EvalSeverity, EvalSource, EvalTargetKind, EvalToolCall, EvalToolCallCountOptions, EvalToolCallMatchOptions, EvalToolCallStatus, EvalToolInputMatchMode, EvalTrace, EvalUsage, EvalUsageSummary, RunEvalOptions, } from "./types.js";
+export type { EvalAgentAdapter, EvalAgentAdapterContext, EvalAgentAdapterResult, EvalAgentInput, EvalAnswerGroundednessMetricOptions, EvalBudgetDeltaSummary, EvalCheckContext, EvalCitation, EvalDataset, EvalDatasetLoadContext, EvalDefinition, EvalDurationSummary, EvalExample, EvalExampleInput, EvalExpect, EvalExpectation, EvalFailedExampleSummary, EvalFlakeSummary, EvalGateFailureSummary, EvalKnowledgeCitationMetricOptions, EvalKnowledgeExpectedSource, EvalKnowledgeMrrMetricOptions, EvalKnowledgeRetrievalMetricOptions, EvalMetric, EvalMetricContext, EvalMetricDeltaSummary, EvalMetricFamily, EvalMetricResult, EvalMetricSummary, EvalMetricThreshold, EvalModelCandidateComparison, EvalModelComparison, EvalModelComparisonConstraint, EvalModelComparisonDecision, EvalModelComparisonMetricName, EvalModelComparisonObjective, EvalModelComparisonOptions, EvalModelReportSummary, EvalRecord, EvalReport, EvalReportComparison, EvalReportComparisonPolicy, EvalReportDatasetMetadata, EvalReportExportConfig, EvalReportMetadata, EvalReportSummary, EvalRetrievedContext, EvalRunProvenance, EvalSeverity, EvalSource, EvalTargetKind, EvalToolAdapter, EvalToolAdapterContext, EvalToolAdapterResult, EvalToolCall, EvalToolCallCountOptions, EvalToolCallMatchOptions, EvalToolCallStatus, EvalToolInput, EvalToolInputMatchMode, EvalTrace, EvalUsage, EvalUsageSummary, RunEvalOptions, } from "./types.js";
 export type { CreateEvalSourceDocumentOptions, EvalEditableField, EvalRun, EvalSourceDocument, EvalSourcePatch, EvalSourceReference, EvalStudioCapability, } from "./studio.js";
 //# sourceMappingURL=index.d.ts.map

package/esm/src/eval/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EACL,yBAAyB,EACzB,gBAAgB,EAChB,0BAA0B,EAC1B,oBAAoB,GACrB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,iCAAiC,EAAE,MAAM,uBAAuB,CAAC;AACnG,OAAO,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AACpF,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EACL,wBAAwB,EACxB,0BAA0B,EAC1B,gBAAgB,EAChB,2BAA2B,EAC3B,wBAAwB,EACxB,4BAA4B,EAC5B,6BAA6B,GAC9B,MAAM,aAAa,CAAC;AAErB,YAAY,EAAE,cAAc,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAChG,YAAY,EAAE,+BAA+B,EAAE,MAAM,aAAa,CAAC;AAEnE,YAAY,EACV,gBAAgB,EAChB,uBAAuB,EACvB,sBAAsB,EACtB,cAAc,EACd,mCAAmC,EACnC,sBAAsB,EACtB,gBAAgB,EAChB,YAAY,EACZ,WAAW,EACX,sBAAsB,EACtB,cAAc,EACd,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,eAAe,EACf,wBAAwB,EACxB,gBAAgB,EAChB,sBAAsB,EACtB,kCAAkC,EAClC,2BAA2B,EAC3B,6BAA6B,EAC7B,mCAAmC,EACnC,UAAU,EACV,iBAAiB,EACjB,sBAAsB,EACtB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,4BAA4B,EAC5B,mBAAmB,EACnB,6BAA6B,EAC7B,2BAA2B,EAC3B,6BAA6B,EAC7B,4BAA4B,EAC5B,0BAA0B,EAC1B,sBAAsB,EACtB,UAAU,EACV,UAAU,EACV,oBAAoB,EACpB,0BAA0B,EAC1B,yBAAyB,EACzB,sBAAsB,EACtB,kBAAkB,EAClB,iBAAiB,EACjB,oBAAoB,EACpB,iBAAiB,EACjB,YAAY,EACZ,UAAU,EACV,cAAc,EACd,YAAY,EACZ,wBAAwB,EACxB,wBAAwB,EACxB,kBAAkB,EAClB,sBAAsB,EACtB,SAAS,EACT,SAAS,EACT,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,+BAA+B,EAC/B,iBAAiB,EACjB,OAAO,EACP,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrE,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EACL,yBAAyB,EACzB,gBAAgB,EAChB,0BAA0B,EAC1B,oBAAoB,GACrB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,iCAAiC,EAAE,MAAM,uBAAuB,CAAC;AACnG,OAAO,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AACpF,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EACL,wBAAwB,EACxB,0BAA0B,EAC1B,gBAAgB,EAChB,2BAA2B,EAC3B,wBAAwB,EACxB,4BAA4B,EAC5B,6BAA6B,EAC7B,uBAAuB,GACxB,MAAM,aAAa,CAAC;AAErB,YAAY,EAAE,cAAc,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAChG,YAAY,EAAE,+BAA+B,EAAE,MAAM,aAAa,CAAC;AAEnE,YAAY,EACV,gBAAgB,EAChB,uBAAuB,EACvB,sBAAsB,EACtB,cAAc,EACd,mCAAmC,EACnC,sBAAsB,EACtB,gBAAgB,EAChB,YAAY,EACZ,WAAW,EACX,sBAAsB,EACtB,cAAc,EACd,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,eAAe,EACf,wBAAwB,EACxB,gBAAgB,EAChB,sBAAsB,EACtB,kCAAkC,EAClC,2BAA2B,EAC3B,6BAA6B,EAC7B,mCAAmC,EACnC,UAAU,EACV,iBAAiB,EACjB,sBAAsB,EACtB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,4BAA4B,EAC5B,mBAAmB,EACnB,6BAA6B,EAC7B,2BAA2B,EAC3B,6BAA6B,EAC7B,4BAA4B,EAC5B,0BAA0B,EAC1B,sBAAsB,EACtB,UAAU,EACV,UAAU,EACV,oBAAoB,EACpB,0BAA0B,EAC1B,yBAAyB,EACzB,sBAAsB,EACtB,kBAAkB,EAClB,iBAAiB,EACjB,oBAAoB,EACpB,iBAAiB,EACjB,YAAY,EACZ,UAAU,EACV,cAAc,EACd,eAAe,EACf,sBAAsB,EACtB,qBAAqB,EACrB,YAAY,EACZ,wBAAwB,EACxB,wBAAwB,EACxB,kBAAkB,EAClB,aAAa,EACb,sBAAsB,EACtB,SAAS,EACT,SAAS,EACT,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,+BAA+B,EAC/B,iBAAiB,EACjB,OAAO,EACP,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,aAAa,CAAC"}

package/esm/src/eval/index.js

@@ -44,7 +44,7 @@
  */
 import "../../_dnt.polyfills.js";
 export { datasets } from "./datasets.js";
-export { evalAgent, isEvalDefinition } from "./factory.js";
+export { evalAgent, evalTool, isEvalDefinition } from "./factory.js";
 export { judges } from "./judges.js";
 export { metrics } from "./metrics.js";
 export { createEvalDatasetMetadata, createEvalReport, EVAL_REPORT_SCHEMA_VERSION, summarizeEvalRecords, } from "./report.js";
@@ -54,4 +54,4 @@ export { createEvalRunProvenance, resolveEvalRunProvenance } from "./provenance.
 export { createEvalRunId } from "./run-id.js";
 export { exportEvalReport, runEval } from "./runner.js";
 export { deriveEvalId, discoverEvals, findEvalById } from "./discovery.js";
-export { createEvalSourceDocument, getEvalEditableFieldSchema, getEvalRunSchema, getEvalSourceDocumentSchema, getEvalSourcePatchSchema, getEvalSourceReferenceSchema, getEvalStudioCapabilitySchema, } from "./studio.js";
+export { createEvalSourceDocument, getEvalEditableFieldSchema, getEvalRunSchema, getEvalSourceDocumentSchema, getEvalSourcePatchSchema, getEvalSourceReferenceSchema, getEvalStudioCapabilitySchema, getEvalTargetKindSchema, } from "./studio.js";

package/esm/src/eval/runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/src/eval/runner.ts"],"names":[],"mappings":"AAEA,OAAO,EAA6B,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAG1E,OAAO,EAKL,KAAK,sBAAsB,EAE5B,MAAM,6BAA6B,CAAC;AAGrC,OAAO,KAAK,EAEV,cAAc,EAGd,sBAAsB,EAGtB,cAAc,EACf,MAAM,YAAY,CAAC;AA8MpB,kFAAkF;AAClF,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,EAC3C,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,sBAAsB,EAAE,GAAG,SAAS,CAAC,CAyB/C;AAmED,6DAA6D;AAC7D,wBAAsB,OAAO,CAC3B,UAAU,EAAE,cAAc,EAC1B,OAAO,EAAE,cAAc,4CA2BxB"}
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/src/eval/runner.ts"],"names":[],"mappings":"AAEA,OAAO,EAA6B,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAG1E,OAAO,EAKL,KAAK,sBAAsB,EAE5B,MAAM,6BAA6B,CAAC;AAGrC,OAAO,KAAK,EAEV,cAAc,EAGd,sBAAsB,EAKtB,cAAc,EACf,MAAM,YAAY,CAAC;AAgRpB,kFAAkF;AAClF,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,EAC3C,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,sBAAsB,EAAE,GAAG,SAAS,CAAC,CAyB/C;AA4FD,6DAA6D;AAC7D,wBAAsB,OAAO,CAC3B,UAAU,EAAE,cAAc,EAC1B,OAAO,EAAE,cAAc,4CA4BxB"}

package/esm/src/eval/runner.js

@@ -6,6 +6,7 @@ import { metrics as runtimeMetrics } from "../metrics/index.js";
 import { createEvalReportExporterRegistry, EvalReportExporterRegistryName, } from "../extensions/eval/index.js";
 import { trace } from "../observability/tracing/api-shim.js";
 import { tryResolve } from "../extensions/contracts.js";
+const UNMAPPED_TOOL_INPUT = Symbol("unmapped-tool-input");
 function normalizeTrace(trace) {
     return {
         events: trace?.events ?? [],
@@ -27,6 +28,46 @@ function normalizeOutput(result) {
         return { text: result.text };
     return result;
 }
+function normalizeToolTargetName(target) {
+    return target.startsWith("tool:") ? target.slice("tool:".length) : target;
+}
+function createDirectToolTraceCall(definition, input, result) {
+    return {
+        ...(result.toolCallId ? { id: result.toolCallId } : {}),
+        name: normalizeToolTargetName(definition.target),
+        status: result.error || result.completed === false ? "error" : "ok",
+        input,
+        output: result.output,
+        ...(result.error ? { error: result.error } : {}),
+        ...(result.durationMs !== undefined ? { metadata: { durationMs: result.durationMs } } : {}),
+    };
+}
+function normalizeToolTrace(definition, input, result) {
+    const trace = normalizeTrace(result.trace);
+    if (trace.toolCalls.length > 0)
+        return trace;
+    return {
+        ...trace,
+        toolCalls: [createDirectToolTraceCall(definition, input, result)],
+    };
+}
+async function runAgentTarget(definition, options, example, repetition) {
+    const adapter = options.adapters.agent;
+    if (!adapter) {
+        throw new Error(`No agent adapter configured for eval target "${definition.target}".`);
+    }
+    return normalizeAdapterResult(await adapter({ definition, example, repetition }));
+}
+async function runToolTarget(definition, options, example, repetition, runId, markInvoked) {
+    const adapter = options.adapters.tool;
+    if (!adapter) {
+        throw new Error(`No tool adapter configured for eval target "${definition.target}".`);
+    }
+    const input = definition.input ? await definition.input(example) : example.input;
+    markInvoked?.();
+    const result = await adapter({ definition, example, repetition, runId, input });
+    return { input, result };
+}
 function isBlockingFailure(record) {
     return [...(record.metrics ?? []), ...(record.checks ?? [])].some((result) => !result.skipped && result.pass === false &&
         (result.severity === "gate" || result.severity === "budget"));
@@ -207,32 +248,55 @@ export async function exportEvalReport(report, config) {
     }
     return results;
 }
-async function runRecord(definition, options, example, repetition) {
+async function runRecord(definition, options, example, repetition, runId) {
     const started = Date.now();
     let result;
+    let toolInput = UNMAPPED_TOOL_INPUT;
+    let toolInvoked = false;
     try {
-        result = normalizeAdapterResult(await options.adapters.agent({ definition, example, repetition }));
+        if (definition.targetKind === "tool") {
+            const toolRun = await runToolTarget(definition, options, example, repetition, runId, () => {
+                toolInvoked = true;
+            });
+            result = toolRun.result;
+            toolInput = toolRun.input;
+        }
+        else {
+            result = await runAgentTarget(definition, options, example, repetition);
+        }
     }
     catch (error) {
         result = {
-            text: "",
+            ...(definition.targetKind === "tool" ? { output: undefined } : { text: "" }),
             completed: false,
             error: error instanceof Error ? error.message : String(error),
         };
     }
-    const output = normalizeOutput(result);
+    const output = definition.targetKind === "tool"
+        ? result.output
+        : normalizeOutput(result);
+    const agentResult = definition.targetKind === "agent"
+        ? result
+        : undefined;
     const record = {
         id: `${example.id}:${repetition}`,
         evalId: definition.id,
         exampleId: example.id,
         repetition,
         input: example.input,
+        ...(definition.targetKind === "tool" && toolInvoked
+            ? { executionInput: toolInput === UNMAPPED_TOOL_INPUT ? example.input : toolInput }
+            : {}),
         output,
         ...(Object.hasOwn(example, "reference") ? { reference: example.reference } : {}),
         metadata: example.metadata ?? {},
-        ...(result.retrievedContext ? { retrievedContext: result.retrievedContext } : {}),
-        ...(result.citations ? { citations: result.citations } : {}),
-        trace: normalizeTrace(result.trace),
+        ...(agentResult?.retrievedContext ? { retrievedContext: agentResult.retrievedContext } : {}),
+        ...(agentResult?.citations ? { citations: agentResult.citations } : {}),
+        trace: definition.targetKind === "tool"
+            ? (toolInvoked
+                ? normalizeToolTrace(definition, toolInput === UNMAPPED_TOOL_INPUT ? example.input : toolInput, result)
+                : normalizeTrace(result.trace))
+            : normalizeTrace(result.trace),
         usage: normalizeUsage(result.usage),
         durationMs: result.durationMs ?? Date.now() - started,
         completed: result.completed ?? !result.error,
@@ -262,20 +326,21 @@ async function runRecord(definition, options, example, repetition) {
 /** Execute an eval locally with injected target adapters. */
 export async function runEval(definition, options) {
     const startedAt = options.now?.() ?? new Date();
+    const runId = options.runId ?? createEvalRunId(startedAt);
     const baseDir = options.baseDir ?? dntShim.Deno.cwd();
     const examples = await definition.dataset.load({ baseDir });
     const dataset = await createEvalDatasetMetadata(definition.dataset, examples);
     const records = [];
     for (const example of examples) {
         for (let repetition = 1; repetition <= definition.repetitions; repetition += 1) {
-            records.push(await runRecord(definition, options, example, repetition));
+            records.push(await runRecord(definition, options, example, repetition, runId));
         }
     }
     const endedAt = options.now?.() ?? new Date();
     const report = createEvalReport({
         definition,
         records,
-        runId: options.runId ?? createEvalRunId(startedAt),
+        runId,
         startedAt,
         endedAt,
         dataset,

package/esm/src/eval/studio.d.ts

@@ -2,8 +2,10 @@ import type { InferSchema } from "../extensions/schema/index.js";
 import type { DiscoveredEval } from "./discovery.js";
 /** Schema for Eval Studio capabilities. */
 export declare const getEvalStudioCapabilitySchema: () => import("../internal-agents/schema.js").Schema<"project.evals.read" | "project.evals.write" | "project.evals.run">;
+/** Schema for an Eval target primitive kind. */
+export declare const getEvalTargetKindSchema: () => import("../internal-agents/schema.js").Schema<"agent" | "tool">;
 /** Schema for an editable Eval source field name. */
-export declare const getEvalEditableFieldSchema: () => import("../internal-agents/schema.js").Schema<"target" | "description" | "metrics" | "name" | "metadata" | "tags" | "check" | "dataset" | "repetitions">;
+export declare const getEvalEditableFieldSchema: () => import("../internal-agents/schema.js").Schema<"target" | "description" | "metrics" | "name" | "input" | "metadata" | "tags" | "check" | "dataset" | "repetitions">;
 /** Schema for an Eval source reference. */
 export declare const getEvalSourceReferenceSchema: () => import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
     filePath: import("../internal-agents/schema.js").Schema<string>;
@@ -119,7 +121,7 @@ export declare const getEvalSourceDocumentSchema: () => import("../internal-agen
     id: import("../internal-agents/schema.js").Schema<string>;
     name: import("../internal-agents/schema.js").Schema<string>;
     description: import("../internal-agents/schema.js").Schema<string | undefined>;
-    targetKind: import("../internal-agents/schema.js").Schema<"agent">;
+    targetKind: import("../internal-agents/schema.js").Schema<"agent" | "tool">;
     target: import("../internal-agents/schema.js").Schema<string>;
     source: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
         filePath: import("../internal-agents/schema.js").Schema<string>;
@@ -153,8 +155,8 @@ export declare const getEvalSourceDocumentSchema: () => import("../internal-agen
     repetitions: import("../internal-agents/schema.js").Schema<number>;
     tags: import("../internal-agents/schema.js").Schema<string[]>;
     metadata: import("../internal-agents/schema.js").Schema<Record<string, unknown>>;
-    editableFields: import("../internal-agents/schema.js").Schema<("target" | "description" | "metrics" | "name" | "metadata" | "tags" | "check" | "dataset" | "repetitions")[]>;
-    dynamicFields: import("../internal-agents/schema.js").Schema<("target" | "description" | "metrics" | "name" | "metadata" | "tags" | "check" | "dataset" | "repetitions")[]>;
+    editableFields: import("../internal-agents/schema.js").Schema<("target" | "description" | "metrics" | "name" | "input" | "metadata" | "tags" | "check" | "dataset" | "repetitions")[]>;
+    dynamicFields: import("../internal-agents/schema.js").Schema<("target" | "description" | "metrics" | "name" | "input" | "metadata" | "tags" | "check" | "dataset" | "repetitions")[]>;
     capabilities: import("../internal-agents/schema.js").Schema<("project.evals.read" | "project.evals.write" | "project.evals.run")[]>;
 }>>;
 /** Schema for a source patch submitted from an Eval editor. */
@@ -205,7 +207,7 @@ export declare const getEvalRunSchema: () => import("../internal-agents/schema.j
     runId: import("../internal-agents/schema.js").Schema<string>;
     evalId: import("../internal-agents/schema.js").Schema<string>;
     status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "running" | "cancelled">;
-    targetKind: import("../internal-agents/schema.js").Schema<"agent">;
+    targetKind: import("../internal-agents/schema.js").Schema<"agent" | "tool">;
     target: import("../internal-agents/schema.js").Schema<string>;
     source: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
         filePath: import("../internal-agents/schema.js").Schema<string>;

package/esm/src/eval/studio.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"studio.d.ts","sourceRoot":"","sources":["../../../src/src/eval/studio.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AACjE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAGrD,2CAA2C;AAC3C,eAAO,MAAM,6BAA6B,yHAEzC,CAAC;AAEF,qDAAqD;AACrD,eAAO,MAAM,0BAA0B,gKActC,CAAC;AAEF,2CAA2C;AAC3C,eAAO,MAAM,4BAA4B;;;;GAMxC,CAAC;AAEF,6DAA6D;AAC7D,eAAO,MAAM,0BAA0B;;;;;GAOtC,CAAC;AAEF,6DAA6D;AAC7D,eAAO,MAAM,0BAA0B;;;;;;;;;;;GAQtC,CAAC;AAEF,4DAA4D;AAC5D,eAAO,MAAM,yBAAyB;;;;;;;;;;;GAarC,CAAC;AAEF,0EAA0E;AAC1E,eAAO,MAAM,6BAA6B;;;;;;;;GAUzC,CAAC;AAEF,qEAAqE;AACrE,eAAO,MAAM,+BAA+B;;;;;;;GAS3C,CAAC;AAEF,8DAA8D;AAC9D,eAAO,MAAM,4BAA4B;;;;;;;;;;;;;;;;;;;;;;GAwBxC,CAAC;AAEF,mEAAmE;AACnE,eAAO,MAAM,kCAAkC;;;;;;;;;GAW9C,CAAC;AAEF,iEAAiE;AACjE,eAAO,MAAM,oCAAoC;;;;;;;GAShD,CAAC;AAEF,kEAAkE;AAClE,eAAO,MAAM,4BAA4B;;;;;GAOxC,CAAC;AAEF,yDAAyD;AACzD,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkBvC,CAAC;AAEF,+DAA+D;AAC/D,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgBpC,CAAC;AAEF,gDAAgD;AAChD,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6B5B,CAAC;AAEF,qEAAqE;AACrE,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,6BAA6B,CAAC,CAAC,CAAC;AACjG,4CAA4C;AAC5C,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC;AAC3F,8CAA8C;AAC9C,MAAM,MAAM,mBAAmB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,4BAA4B,CAAC,CAAC,CAAC;AAC/F,4CAA4C;AAC5C,MAAM,MAAM,kBAAkB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,2BAA2B,CAAC,CAAC,CAAC;AAC7F,mDAAmD;AACnD,MAAM,MAAM,eAAe,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,wBAAwB,CAAC,CAAC,CAAC;AACvF,oCAAoC;AACpC,MAAM,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC,CAAC;AAEvE,4EAA4E;AAC5E,MAAM,WAAW,+BAA+B;IAC9C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,oBAAoB,EAAE,CAAC;CACvC;AAwDD,8EAA8E;AAC9E,wBAAgB,wBAAwB,CACtC,UAAU,EAAE,cAAc,EAC1B,OAAO,GAAE,+BAAoC,GAC5C,kBAAkB,CAyBpB"}
+{"version":3,"file":"studio.d.ts","sourceRoot":"","sources":["../../../src/src/eval/studio.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AACjE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAGrD,2CAA2C;AAC3C,eAAO,MAAM,6BAA6B,yHAEzC,CAAC;AAEF,gDAAgD;AAChD,eAAO,MAAM,uBAAuB,uEAA0D,CAAC;AAE/F,qDAAqD;AACrD,eAAO,MAAM,0BAA0B,0KAetC,CAAC;AAEF,2CAA2C;AAC3C,eAAO,MAAM,4BAA4B;;;;GAMxC,CAAC;AAEF,6DAA6D;AAC7D,eAAO,MAAM,0BAA0B;;;;;GAOtC,CAAC;AAEF,6DAA6D;AAC7D,eAAO,MAAM,0BAA0B;;;;;;;;;;;GAQtC,CAAC;AAEF,4DAA4D;AAC5D,eAAO,MAAM,yBAAyB;;;;;;;;;;;GAarC,CAAC;AAEF,0EAA0E;AAC1E,eAAO,MAAM,6BAA6B;;;;;;;;GAUzC,CAAC;AAEF,qEAAqE;AACrE,eAAO,MAAM,+BAA+B;;;;;;;GAS3C,CAAC;AAEF,8DAA8D;AAC9D,eAAO,MAAM,4BAA4B;;;;;;;;;;;;;;;;;;;;;;GAwBxC,CAAC;AAEF,mEAAmE;AACnE,eAAO,MAAM,kCAAkC;;;;;;;;;GAW9C,CAAC;AAEF,iEAAiE;AACjE,eAAO,MAAM,oCAAoC;;;;;;;GAShD,CAAC;AAEF,kEAAkE;AAClE,eAAO,MAAM,4BAA4B;;;;;GAOxC,CAAC;AAEF,yDAAyD;AACzD,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkBvC,CAAC;AAEF,+DAA+D;AAC/D,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgBpC,CAAC;AAEF,gDAAgD;AAChD,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6B5B,CAAC;AAEF,qEAAqE;AACrE,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,6BAA6B,CAAC,CAAC,CAAC;AACjG,4CAA4C;AAC5C,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC;AAC3F,8CAA8C;AAC9C,MAAM,MAAM,mBAAmB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,4BAA4B,CAAC,CAAC,CAAC;AAC/F,4CAA4C;AAC5C,MAAM,MAAM,kBAAkB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,2BAA2B,CAAC,CAAC,CAAC;AAC7F,mDAAmD;AACnD,MAAM,MAAM,eAAe,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,wBAAwB,CAAC,CAAC,CAAC;AACvF,oCAAoC;AACpC,MAAM,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC,CAAC;AAEvE,4EAA4E;AAC5E,MAAM,WAAW,+BAA+B;IAC9C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,oBAAoB,EAAE,CAAC;CACvC;AAwDD,8EAA8E;AAC9E,wBAAgB,wBAAwB,CACtC,UAAU,EAAE,cAAc,EAC1B,OAAO,GAAE,+BAAoC,GAC5C,kBAAkB,CA4BpB"}

package/esm/src/eval/studio.js

@@ -1,6 +1,8 @@
 import { defineSchema } from "../schemas/index.js";
 /** Schema for Eval Studio capabilities. */
 export const getEvalStudioCapabilitySchema = defineSchema((v) => v.enum(["project.evals.read", "project.evals.write", "project.evals.run"]));
+/** Schema for an Eval target primitive kind. */
+export const getEvalTargetKindSchema = defineSchema((v) => v.enum(["agent", "tool"]));
 /** Schema for an editable Eval source field name. */
 export const getEvalEditableFieldSchema = defineSchema((v) => v.enum([
     "name",
@@ -11,6 +13,7 @@ export const getEvalEditableFieldSchema = defineSchema((v) => v.enum([
     "tags",
     "metadata",
     "metrics",
+    "input",
     "check",
 ]));
 /** Schema for an Eval source reference. */
@@ -123,7 +126,7 @@ export const getEvalSourceDocumentSchema = defineSchema((v) => v.object({
     id: v.string(),
     name: v.string(),
     description: v.string().optional(),
-    targetKind: v.literal("agent"),
+    targetKind: getEvalTargetKindSchema(),
     target: v.string(),
     source: getEvalSourceReferenceSchema(),
     dataset: getEvalSourceDatasetSchema(),
@@ -157,7 +160,7 @@ export const getEvalRunSchema = defineSchema((v) => v.object({
     runId: v.string(),
     evalId: v.string(),
     status: v.enum(["pending", "running", "waiting", "completed", "failed", "cancelled"]),
-    targetKind: v.literal("agent"),
+    targetKind: getEvalTargetKindSchema(),
     target: v.string(),
     source: getEvalSourceReferenceSchema().optional(),
     summary: v.object({
@@ -230,7 +233,10 @@ function createSourceMetric(metric) {
 /** Create the normalized Eval document Studio can list, inspect, and edit. */
 export function createEvalSourceDocument(discovered, options = {}) {
     const { definition } = discovered;
-    const dynamicFields = definition.check ? ["check"] : [];
+    const dynamicFields = [
+        ...(definition.input ? ["input"] : []),
+        ...(definition.check ? ["check"] : []),
+    ];
     return getEvalSourceDocumentSchema().parse({
         kind: "eval-source-document",
         id: discovered.id,