veryfront 0.1.1059 → 0.1.1062

package/esm/_dnt.polyfills.d.ts

@@ -109,6 +109,17 @@ interface ImportMetaPonyfill extends ImportMetaPonyfillCommonjs, ImportMetaPonyf
 export declare let import_meta_ponyfill_commonjs: ImportMetaPonyfillCommonjs;
 export declare let import_meta_ponyfill_esmodule: ImportMetaPonyfillEsmodule;
 export declare let import_meta_ponyfill: ImportMetaPonyfill;
+declare global {
+    interface Object {
+        /**
+         * Determines whether an object has a property with the specified name.
+         * @param o An object.
+         * @param v A property name.
+         */
+        hasOwn(o: object, v: PropertyKey): boolean;
+    }
+}
+export {};
 declare global {
     interface Array<T> {
         /**
@@ -158,17 +169,6 @@ declare global {
     }
 }
 export {};
-declare global {
-    interface Object {
-        /**
-         * Determines whether an object has a property with the specified name.
-         * @param o An object.
-         * @param v A property name.
-         */
-        hasOwn(o: object, v: PropertyKey): boolean;
-    }
-}
-export {};
 declare global {
     interface PromiseConstructor {
         /**

package/esm/_dnt.polyfills.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"_dnt.polyfills.d.ts","sourceRoot":"","sources":["../src/_dnt.polyfills.ts"],"names":[],"mappings":"AAAA,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,KAAK;QACb,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB;CACF;AAED,OAAO,EAAE,CAAC;AAEV,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,gBAAgB;QACxB,SAAS,CAAC,CAAC,EACP,mBAAmB,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,GAC7F,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;QAEhB,SAAS,CAAC,CAAC,EAAE,CAAC,EACV,mBAAmB,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,EAClE,KAAK,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,EAC/B,OAAO,CAAC,EAAE,GAAG,GACd,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;KAC1B;CACF;AAoID,OAAO,EAAE,CAAC,CAAA;;;;;;;;;;;GAWP;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,OAAO,EAAgC,KAAK,GAAG,EAAE,MAAM,UAAU,CAAC;AAGlE,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,UAAU;QAClB;;;;;;;;;;;;;;WAcG;QACH,GAAG,EAAE,MAAM,CAAC;QACZ;;;;;;;;;;;;WAYG;QACH,OAAO,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,GAAG,GAAG,SAAS,GAAG,MAAM,CAAC;QACtE;;;;;;;;WAQG;QACH,IAAI,EAAE,OAAO,CAAC;QAEd;;;;;;;;;;;;WAYG;QACH,QAAQ,EAAE,MAAM,CAAC;QAEjB;;;;;;;;;;;;WAYG;QACH,OAAO,EAAE,MAAM,CAAC;KACjB;CACF;AAED,KAAK,WAAW,GAAG,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC;AACpD,KAAK,UAAU,GAAG,WAAW,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;AACnD,UAAU,0BAA0B;IAClC,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,GAAG,UAAU,CAAC;CACxD;AACD,UAAU,0BAA0B;IAClC,CAAC,UAAU,EAAE,UAAU,GAAG,UAAU,CAAC;CACtC;AACD,UAAU,kBACR,SAAQ,0BAA0B,EAAE,0BAA0B;CAC/D;AAiBD,eAAO,IAAI,6BAA6B,EA2BnC,0BAA0B,CAAC;AAMhC,eAAO,IAAI,6BAA6B,EA4DnC,0BAA0B,CAAC;AAMhC,eAAO,IAAI,oBAAoB,EAoB1B,kBAAkB,CAAC;AAExB,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,KAAK,CAAC,CAAC;QACf;;;;;;;;WAQG;QACH,QAAQ,CAAC,CAAC,SAAS,CAAC,EAClB,SAAS,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC,EACxE,OAAO,CAAC,EAAE,GAAG,GACZ,CAAC,GAAG,SAAS,CAAC;QACjB,QAAQ,CACN,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,OAAO,EACzD,OAAO,CAAC,EAAE,GAAG,GACZ,CAAC,GAAG,SAAS,CAAC;QAEjB;;;;;;;;WAQG;QACH,aAAa,CACX,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,OAAO,EACzD,OAAO,CAAC,EAAE,GAAG,GACZ,MAAM,CAAC;KACX;IACD,UAAU,UAAU;QAClB;;;;;;;;WAQG;QACH,QAAQ,CAAC,CAAC,SAAS,MAAM,EACvB,SAAS,EAAE,CACP,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,UAAU,KAChB,KAAK,IAAI,CAAC,EACf,OAAO,CAAC,EAAE,GAAG,GACZ,CAAC,GAAG,SAAS,CAAC;QACjB,QAAQ,CACJ,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,KAAK,OAAO,EACvE,OAAO,CAAC,EAAE,GAAG,GACd,MAAM,GAAG,SAAS,CAAC;QAEtB;;;;;;;;WAQG;QACH,aAAa,CACT,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,KAAK,OAAO,EACvE,OAAO,CAAC,EAAE,GAAG,GACd,MAAM,CAAC;KACX;CACF;AA4CD,OAAO,EAAE,CAAC;AAgBV,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,MAAM;QACd;;;;WAIG;QACH,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC;KAC5C;CACF;AAED,OAAO,EAAE,CAAC;AACV,OAAO,CAAC,MAAM,CAAC;IAEb,UAAU,kBAAkB;QAC1B;;;WAGG;QACH,aAAa,CAAC,CAAC,KAAK;YAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;YAAC,OAAO,EAAE,CAAC,KAAK,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC;YAAC,MAAM,EAAE,CAAC,MAAM,CAAC,EAAE,GAAG,KAAK,IAAI,CAAA;SAAE,CAAC;KAC3H;CACF"}
+{"version":3,"file":"_dnt.polyfills.d.ts","sourceRoot":"","sources":["../src/_dnt.polyfills.ts"],"names":[],"mappings":"AAAA,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,KAAK;QACb,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB;CACF;AAED,OAAO,EAAE,CAAC;AAEV,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,gBAAgB;QACxB,SAAS,CAAC,CAAC,EACP,mBAAmB,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,GAC7F,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;QAEhB,SAAS,CAAC,CAAC,EAAE,CAAC,EACV,mBAAmB,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,EAClE,KAAK,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,EAC/B,OAAO,CAAC,EAAE,GAAG,GACd,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;KAC1B;CACF;AAoID,OAAO,EAAE,CAAC,CAAA;;;;;;;;;;;GAWP;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,OAAO,EAAgC,KAAK,GAAG,EAAE,MAAM,UAAU,CAAC;AAGlE,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,UAAU;QAClB;;;;;;;;;;;;;;WAcG;QACH,GAAG,EAAE,MAAM,CAAC;QACZ;;;;;;;;;;;;WAYG;QACH,OAAO,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,GAAG,GAAG,SAAS,GAAG,MAAM,CAAC;QACtE;;;;;;;;WAQG;QACH,IAAI,EAAE,OAAO,CAAC;QAEd;;;;;;;;;;;;WAYG;QACH,QAAQ,EAAE,MAAM,CAAC;QAEjB;;;;;;;;;;;;WAYG;QACH,OAAO,EAAE,MAAM,CAAC;KACjB;CACF;AAED,KAAK,WAAW,GAAG,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC;AACpD,KAAK,UAAU,GAAG,WAAW,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;AACnD,UAAU,0BAA0B;IAClC,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,GAAG,UAAU,CAAC;CACxD;AACD,UAAU,0BAA0B;IAClC,CAAC,UAAU,EAAE,UAAU,GAAG,UAAU,CAAC;CACtC;AACD,UAAU,kBACR,SAAQ,0BAA0B,EAAE,0BAA0B;CAC/D;AAiBD,eAAO,IAAI,6BAA6B,EA2BnC,0BAA0B,CAAC;AAMhC,eAAO,IAAI,6BAA6B,EA4DnC,0BAA0B,CAAC;AAMhC,eAAO,IAAI,oBAAoB,EAoB1B,kBAAkB,CAAC;AAgBxB,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,MAAM;QACd;;;;WAIG;QACH,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC;KAC5C;CACF;AAED,OAAO,EAAE,CAAC;AAEV,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,KAAK,CAAC,CAAC;QACf;;;;;;;;WAQG;QACH,QAAQ,CAAC,CAAC,SAAS,CAAC,EAClB,SAAS,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC,EACxE,OAAO,CAAC,EAAE,GAAG,GACZ,CAAC,GAAG,SAAS,CAAC;QACjB,QAAQ,CACN,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,OAAO,EACzD,OAAO,CAAC,EAAE,GAAG,GACZ,CAAC,GAAG,SAAS,CAAC;QAEjB;;;;;;;;WAQG;QACH,aAAa,CACX,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,OAAO,EACzD,OAAO,CAAC,EAAE,GAAG,GACZ,MAAM,CAAC;KACX;IACD,UAAU,UAAU;QAClB;;;;;;;;WAQG;QACH,QAAQ,CAAC,CAAC,SAAS,MAAM,EACvB,SAAS,EAAE,CACP,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,UAAU,KAChB,KAAK,IAAI,CAAC,EACf,OAAO,CAAC,EAAE,GAAG,GACZ,CAAC,GAAG,SAAS,CAAC;QACjB,QAAQ,CACJ,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,KAAK,OAAO,EACvE,OAAO,CAAC,EAAE,GAAG,GACd,MAAM,GAAG,SAAS,CAAC;QAEtB;;;;;;;;WAQG;QACH,aAAa,CACT,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,KAAK,OAAO,EACvE,OAAO,CAAC,EAAE,GAAG,GACd,MAAM,CAAC;KACX;CACF;AA4CD,OAAO,EAAE,CAAC;AACV,OAAO,CAAC,MAAM,CAAC;IAEb,UAAU,kBAAkB;QAC1B;;;WAGG;QACH,aAAa,CAAC,CAAC,KAAK;YAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;YAAC,OAAO,EAAE,CAAC,KAAK,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC;YAAC,MAAM,EAAE,CAAC,MAAM,CAAC,EAAE,GAAG,KAAK,IAAI,CAAA;SAAE,CAAC;KAC3H;CACF"}

package/esm/_dnt.polyfills.js

@@ -222,6 +222,20 @@ export let import_meta_ponyfill = ((...args) => {
     //@ts-ignore
     return import_meta_ponyfill(...args);
 });
+// https://github.com/tc39/proposal-accessible-object-hasownproperty/blob/main/polyfill.js
+if (!Object.hasOwn) {
+    Object.defineProperty(Object, "hasOwn", {
+        value: function (object, property) {
+            if (object == null) {
+                throw new TypeError("Cannot convert undefined or null to object");
+            }
+            return Object.prototype.hasOwnProperty.call(Object(object), property);
+        },
+        configurable: true,
+        enumerable: false,
+        writable: true,
+    });
+}
 function findLastIndex(self, callbackfn, that) {
     const boundFunc = that === undefined ? callbackfn : callbackfn.bind(that);
     let index = self.length - 1;
@@ -258,20 +272,6 @@ if (!Uint8Array.prototype.findLast) {
         return findLast(this, callbackfn, that);
     };
 }
-// https://github.com/tc39/proposal-accessible-object-hasownproperty/blob/main/polyfill.js
-if (!Object.hasOwn) {
-    Object.defineProperty(Object, "hasOwn", {
-        value: function (object, property) {
-            if (object == null) {
-                throw new TypeError("Cannot convert undefined or null to object");
-            }
-            return Object.prototype.hasOwnProperty.call(Object(object), property);
-        },
-        configurable: true,
-        enumerable: false,
-        writable: true,
-    });
-}
 // https://github.com/tc39/proposal-promise-with-resolvers/blob/3a78801e073e99217dbeb2c43ba7212f3bdc8b83/polyfills.js#L1C1-L9C2
 if (Promise.withResolvers === undefined) {
     Promise.withResolvers = () => {

package/esm/cli/commands/eval/command.d.ts

@@ -1,6 +1,8 @@
 import type { AgentResponse } from "../../../src/agent/index.js";
 import type { VeryfrontConfig } from "../../../src/config/index.js";
-import type { DiscoveredEval, EvalModelComparison, EvalModelComparisonOptions, EvalReport, EvalReportComparison, EvalReportExportConfig, EvalToolCall } from "../../../src/eval/index.js";
+import { type Tool, type ToolExecutionContext } from "../../../src/tool/index.js";
+import type { DiscoveredEval, EvalModelComparison, EvalModelComparisonOptions, EvalReport, EvalReportComparison, EvalReportExportConfig, EvalToolAdapterContext, EvalToolCall } from "../../../src/eval/index.js";
+import { type EvalReportExporterRegistry, type EvalReportExportRedaction } from "../../../src/extensions/eval/index.js";
 import type { EvalArgs } from "./handler.js";
 export interface EvalOptions extends EvalArgs {
 }
@@ -74,12 +76,21 @@ export declare function createEvalMarkdownReport(report: EvalReport, baseline?:
 export declare function createJunitXml(report: EvalReport): string;
 export declare function writeEvalArtifacts(report: EvalReport, paths: EvalArtifactPaths, baseline?: EvalReportComparison): Promise<void>;
 export declare function createEvalExitCode(report: EvalReport, baseline?: EvalReportComparison): 0 | 1;
+export declare function resolveToolTargetId(target: string): string;
 type EvalRuntimeAuthConfig = Pick<VeryfrontConfig, "projectSlug" | "fs"> & {
     projectSlug?: string;
 };
 export declare function hydrateEvalRuntimeAuth(projectDir: string, config: EvalRuntimeAuthConfig | null | undefined): Promise<import("../../shared/runtime-auth.js").RuntimeAuthContext>;
 export declare function normalizeUsage(response: AgentResponse): {};
 export declare function normalizeToolCalls(response: AgentResponse): EvalToolCall[];
+export declare function createToolAdapter(tool: Tool, baseContext?: ToolExecutionContext): (context: EvalToolAdapterContext) => Promise<{
+    error?: string | undefined;
+    output: any;
+    toolCallId: string;
+    durationMs: number;
+    completed: boolean;
+}>;
+export declare function createEvalCliExportConfig(evalItem: DiscoveredEval, options: EvalOptions, projectDir: string, artifactPaths: EvalArtifactPaths, registry: EvalReportExporterRegistry, config?: EvalRuntimeAuthConfig | null): EvalReportExportConfig | undefined;
 type EvalModelComparisonConfig = {
     baselineModel: string;
     candidateModels: string[];
@@ -90,6 +101,8 @@ type ResolvedEvalModelComparisonConfig = {
     config: EvalModelComparisonConfig;
     policy: EvalModelComparisonPolicy;
 };
+export declare function resolveEvalExporterIds(options: Pick<EvalOptions, "exporters">): string[];
+export declare function resolveEvalExportRedactionFromEnv(): EvalReportExportRedaction;
 export declare function evalCommand(options: EvalOptions): Promise<void>;
 export {};
 //# sourceMappingURL=command.d.ts.map

package/esm/cli/commands/eval/command.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EAEpB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AA2BpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,+BAA+B,GAAG;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAOF,KAAK,6BAA6B,GAAG;IACnC,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AAoDnF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhF;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAO5E;AAMD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CASxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAgBrB;AA8GD,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,+BAA+B,GAC5C,UAAU,CAiBZ;AAWD,wBAAsB,2BAA2B,CAC/C,cAAc,EAAE,MAAM,EACtB,OAAO,GAAE,6BAAkC,GAC1C,OAAO,CAAC,+BAA+B,GAAG,SAAS,CAAC,CAgDtD;AAED,wBAAsB,8BAA8B,CAClD,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,GACnC,OAAO,CAAC,UAAU,CAAC,CAerB;AAED,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,UAAU,CAAC,CAGrB;AA8FD,wBAAsB,6BAA6B,CACjD,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,yBAAyB,CAAC,CA0CpC;AAED,wBAAsB,uCAAuC,CAC3D,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,iCAAiC,GAAG,SAAS,CAAC,CAKxD;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,EACrB,MAAM,GAAE,yBAA8B,GACrC,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAE9E;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAkED,uEAAuE;AACvE,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,MAAM,CAkFR;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAQf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,CAAC,GAAG,CAAC,CAEP;AAMD,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,MA0DrD;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AA+FD,KAAK,yBAAyB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,KAAK,iCAAiC,GAAG;IACvC,MAAM,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AA8LF,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CA0JrE"}
+{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAgC,KAAK,IAAI,EAAE,KAAK,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAChH,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EAEpB,sBAAsB,EACtB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AAGpC,OAAO,EAEL,KAAK,0BAA0B,EAE/B,KAAK,yBAAyB,EAC/B,MAAM,uCAAuC,CAAC;AA4B/C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,+BAA+B,GAAG;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAOF,KAAK,6BAA6B,GAAG;IACnC,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AA8DnF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhF;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAO5E;AAMD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CASxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAgBrB;AA8GD,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,+BAA+B,GAC5C,UAAU,CAiBZ;AAWD,wBAAsB,2BAA2B,CAC/C,cAAc,EAAE,MAAM,EACtB,OAAO,GAAE,6BAAkC,GAC1C,OAAO,CAAC,+BAA+B,GAAG,SAAS,CAAC,CAgDtD;AAED,wBAAsB,8BAA8B,CAClD,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,GACnC,OAAO,CAAC,UAAU,CAAC,CAerB;AAED,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,UAAU,CAAC,CAGrB;AA8FD,wBAAsB,6BAA6B,CACjD,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,yBAAyB,CAAC,CA0CpC;AAED,wBAAsB,uCAAuC,CAC3D,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,iCAAiC,GAAG,SAAS,CAAC,CAKxD;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,EACrB,MAAM,GAAE,yBAA8B,GACrC,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAE9E;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAkED,uEAAuE;AACvE,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,MAAM,CAkFR;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAQf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,CAAC,GAAG,CAAC,CAEP;AAMD,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAaD,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,MA0DrD;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AAqCD,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,GAAE,oBAAyB,IACpE,SAAS,sBAAsB;;;;;;GAiB9C;AAoDD,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,WAAW,EACpB,UAAU,EAAE,MAAM,EAClB,aAAa,EAAE,iBAAiB,EAChC,QAAQ,EAAE,0BAA0B,EACpC,MAAM,CAAC,EAAE,qBAAqB,GAAG,IAAI,GACpC,sBAAsB,GAAG,SAAS,CAkBpC;AAED,KAAK,yBAAyB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,KAAK,iCAAiC,GAAG;IACvC,MAAM,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AAuBF,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,GAAG,MAAM,EAAE,CAOxF;AA2BD,wBAAgB,iCAAiC,IAAI,yBAAyB,CAuB7E;AA8OD,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAgMrE"}

package/esm/cli/commands/eval/command.js

@@ -3,14 +3,28 @@
  */
 import * as dntShim from "../../../_dnt.shims.js";
 import { dirname, isAbsolute, join, relative, resolve } from "../../../deps/jsr.io/@std/path/1.1.4/mod.js";
+import { isErroredToolExecutionResult } from "../../../src/tool/index.js";
 import { createProjectDiscoveryConfig, discoverAll } from "../../../src/discovery/index.js";
+import { orchestrateExtensions } from "../../../src/extensions/index.js";
+import { createEvalReportExporterRegistry, EvalReportExporterRegistryName, } from "../../../src/extensions/eval/index.js";
+import { createLLMProviderRegistry, LLMProviderRegistryName } from "../../../src/extensions/llm/index.js";
 import { compareEvalModelReports, compareEvalReports, createEvalModelComparisonMarkdown, createEvalRunId, discoverEvals, EVAL_REPORT_SCHEMA_VERSION, exportEvalReport, resolveEvalRunProvenance, runEval, } from "../../../src/eval/index.js";
 import { getCurrentVeryfrontCloudContext, getVeryfrontCloudBootstrap, runWithVeryfrontCloudContextAsync, } from "../../../src/provider/index.js";
 import { applyRuntimeAuthContext } from "../../shared/runtime-auth.js";
 import { cliLogger, exitProcess, VERSION } from "../../utils/index.js";
 import { createErrorEnvelope, createSuccessEnvelope, isJsonMode, outputJson, } from "../../shared/json-output.js";
 import { withProjectSourceContext } from "../../shared/project-source-context.js";
+import { createEvalCliBuiltinExtensions } from "../../../src/extensions/builtin-extensions.js";
 const GATEWAY_BILLING_GROUP_USAGE_NOT_READY_CODE = "gateway_billing_group_usage_not_ready";
+const ENV_EVAL_EXPORTERS = "VERYFRONT_EVAL_EXPORTERS";
+const ENV_EVAL_EXPORT = "VERYFRONT_EVAL_EXPORT";
+const ENV_EVAL_EXPORT_INCLUDE_INPUTS = "VERYFRONT_EVAL_EXPORT_INCLUDE_INPUTS";
+const ENV_EVAL_EXPORT_INCLUDE_OUTPUTS = "VERYFRONT_EVAL_EXPORT_INCLUDE_OUTPUTS";
+const ENV_EVAL_EXPORT_INCLUDE_REFERENCES = "VERYFRONT_EVAL_EXPORT_INCLUDE_REFERENCES";
+const ENV_EVAL_EXPORT_INCLUDE_TRACES = "VERYFRONT_EVAL_EXPORT_INCLUDE_TRACES";
+const ENV_EVAL_EXPORT_INCLUDE_METRIC_EVIDENCE = "VERYFRONT_EVAL_EXPORT_INCLUDE_METRIC_EVIDENCE";
+const ENV_EVAL_EXPORT_INCLUDE_METRIC_EXPLANATIONS = "VERYFRONT_EVAL_EXPORT_INCLUDE_METRIC_EXPLANATIONS";
+const ENV_EVAL_EXPORT_METADATA_ALLOWLIST = "VERYFRONT_EVAL_EXPORT_METADATA_ALLOWLIST";
 // Gateway usage capture is eventually consistent after model streams close.
 const DEFAULT_GATEWAY_BILLING_FINALIZE_RETRY_DELAYS_MS = [
     500,
@@ -624,6 +638,9 @@ export function createEvalExitCode(report, baseline) {
 function resolveAgentTargetId(target) {
     return target.startsWith("agent:") ? target.slice("agent:".length) : target;
 }
+export function resolveToolTargetId(target) {
+    return target.startsWith("tool:") ? target.slice("tool:".length) : target;
+}
 function resolveEvalRuntimeProjectSlug(config) {
     return config?.projectSlug ?? config?.fs?.veryfront?.projectSlug;
 }
@@ -633,6 +650,14 @@ export async function hydrateEvalRuntimeAuth(projectDir, config) {
         projectSlug: resolveEvalRuntimeProjectSlug(config),
     });
 }
+function createEvalToolExecutionContext(config) {
+    const projectSlug = resolveEvalRuntimeProjectSlug(config);
+    const authToken = dntShim.Deno.env.get("VERYFRONT_API_TOKEN");
+    return {
+        ...(projectSlug ? { projectSlug } : {}),
+        ...(authToken ? { authToken } : {}),
+    };
+}
 export function normalizeUsage(response) {
     return response.usage
         ? {
@@ -705,6 +730,19 @@ export function normalizeToolCalls(response) {
             : {}),
     }));
 }
+function getToolExecutionErrorMessage(output) {
+    if (!isErroredToolExecutionResult(output))
+        return undefined;
+    if (isRecord(output) && typeof output.error === "string")
+        return output.error;
+    if (isRecord(output) && isRecord(output.output) && typeof output.output.error === "string") {
+        return output.output.error;
+    }
+    return "Tool execution returned an error result.";
+}
+function createEvalToolCallId(toolId, context) {
+    return `eval-${toolId}-${context.example.id}-${context.repetition}-${dntShim.crypto.randomUUID()}`;
+}
 function createAgentAdapter(agent, options) {
     return async ({ example }) => {
         const started = Date.now();
@@ -726,6 +764,25 @@ function createAgentAdapter(agent, options) {
         };
     };
 }
+export function createToolAdapter(tool, baseContext = {}) {
+    return async (context) => {
+        const started = Date.now();
+        const toolCallId = createEvalToolCallId(tool.id, context);
+        const output = await tool.execute(context.input, {
+            ...baseContext,
+            runId: context.runId,
+            toolCallId,
+        });
+        const error = getToolExecutionErrorMessage(output);
+        return {
+            output,
+            toolCallId,
+            durationMs: Date.now() - started,
+            completed: !error,
+            ...(error ? { error } : {}),
+        };
+    };
+}
 function listEvals(evals, projectDir) {
     return evals.map((item) => ({
         id: item.id,
@@ -760,24 +817,102 @@ function printReport(report, baseline) {
         }
     }
 }
-function createEvalCliExportConfig(evalItem, options, projectDir, artifactPaths) {
-    if (options.exporters.length === 0)
+export function createEvalCliExportConfig(evalItem, options, projectDir, artifactPaths, registry, config) {
+    const exporterIds = resolveEvalExporterIds(options);
+    if (exporterIds.length === 0)
         return undefined;
+    const projectReference = resolveEvalRuntimeProjectSlug(config);
     return {
-        exporterIds: options.exporters,
+        registry,
+        exporterIds,
         context: {
             evalId: evalItem.definition.id,
+            ...(projectReference ? { projectReference } : {}),
             sourcePath: displaySourcePath(evalItem.filePath, projectDir),
             reportPath: options.report ?? artifactPaths.summary,
             tags: evalItem.definition.tags,
             metadata: evalItem.definition.metadata,
-            redaction: {},
+            redaction: resolveEvalExportRedactionFromEnv(),
         },
     };
 }
 function uniqueValues(values) {
     return Array.from(new Set(values.map((value) => value.trim()).filter(Boolean)));
 }
+function readEvalCliEnv(name) {
+    try {
+        const value = dntShim.Deno.env.get(name);
+        return value && value.trim().length > 0 ? value : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function parseEvalExporterList(value) {
+    if (!value)
+        return [];
+    return value
+        .split(/[,\s]+/)
+        .map((entry) => entry.trim())
+        .filter((entry) => entry.length > 0);
+}
+export function resolveEvalExporterIds(options) {
+    if (options.exporters.length > 0)
+        return uniqueValues(options.exporters);
+    const envExporters = parseEvalExporterList(readEvalCliEnv(ENV_EVAL_EXPORTERS));
+    if (envExporters.length > 0)
+        return uniqueValues(envExporters);
+    return uniqueValues(parseEvalExporterList(readEvalCliEnv(ENV_EVAL_EXPORT)));
+}
+function parseEvalExportBooleanEnv(name) {
+    const value = readEvalCliEnv(name)?.toLowerCase();
+    if (value === undefined)
+        return undefined;
+    if (["1", "true", "yes", "on"].includes(value))
+        return true;
+    if (["0", "false", "no", "off"].includes(value))
+        return false;
+    return false;
+}
+function assignEvalExportBooleanEnv(redaction, key, envName) {
+    const value = parseEvalExportBooleanEnv(envName);
+    if (value !== undefined)
+        redaction[key] = value;
+}
+export function resolveEvalExportRedactionFromEnv() {
+    const redaction = {};
+    assignEvalExportBooleanEnv(redaction, "includeInputs", ENV_EVAL_EXPORT_INCLUDE_INPUTS);
+    assignEvalExportBooleanEnv(redaction, "includeOutputs", ENV_EVAL_EXPORT_INCLUDE_OUTPUTS);
+    assignEvalExportBooleanEnv(redaction, "includeReferences", ENV_EVAL_EXPORT_INCLUDE_REFERENCES);
+    assignEvalExportBooleanEnv(redaction, "includeTraces", ENV_EVAL_EXPORT_INCLUDE_TRACES);
+    assignEvalExportBooleanEnv(redaction, "includeMetricEvidence", ENV_EVAL_EXPORT_INCLUDE_METRIC_EVIDENCE);
+    assignEvalExportBooleanEnv(redaction, "includeMetricExplanations", ENV_EVAL_EXPORT_INCLUDE_METRIC_EXPLANATIONS);
+    const metadataAllowlist = parseEvalExporterList(readEvalCliEnv(ENV_EVAL_EXPORT_METADATA_ALLOWLIST));
+    if (metadataAllowlist.length > 0)
+        redaction.metadataAllowlist = uniqueValues(metadataAllowlist);
+    return redaction;
+}
+function createEvalCliPrimeContracts() {
+    const exporterRegistry = createEvalReportExporterRegistry();
+    return {
+        exporterRegistry,
+        contracts: {
+            [LLMProviderRegistryName]: createLLMProviderRegistry(),
+            [EvalReportExporterRegistryName]: exporterRegistry,
+        },
+    };
+}
+async function setupEvalCliExtensions(projectDir, config, exporterIds) {
+    const primeContracts = createEvalCliPrimeContracts();
+    const loader = await orchestrateExtensions({
+        projectDir,
+        config,
+        logger: cliLogger.component("eval-extensions"),
+        primeContracts: primeContracts.contracts,
+        builtinExtensions: createEvalCliBuiltinExtensions(exporterIds),
+    });
+    return { loader, exporterRegistry: primeContracts.exporterRegistry };
+}
 function resolveEvalModelComparisonConfig(options) {
     const candidateModels = uniqueValues(options.candidateModels);
     const baselineModel = options.baselineModel?.trim();
@@ -828,7 +963,7 @@ async function runEvalModelComparison(input) {
         const modelPaths = paths.models[model];
         const modelOptions = { ...input.options, model, report: undefined };
         const modelRunId = `${runId}_${sanitizeModelIdForPath(model)}`;
-        const exportConfig = createEvalCliExportConfig(input.evalItem, modelOptions, input.projectDir, modelPaths);
+        const exportConfig = createEvalCliExportConfig(input.evalItem, modelOptions, input.projectDir, modelPaths, input.exporterRegistry, input.projectReference ? { projectSlug: input.projectReference } : undefined);
         const finalizedReport = await runEvalWithGatewayBillingGroup(modelRunId, () => runEval(input.evalItem.definition, {
             baseDir: input.options.datasetBase ?? input.projectDir,
             runId: modelRunId,
@@ -873,7 +1008,7 @@ async function runEvalModelComparison(input) {
         if (input.options.report)
             cliLogger.info(`Report: ${input.options.report}`);
     }
-    exitProcess(createEvalModelComparisonExitCode(reports));
+    return createEvalModelComparisonExitCode(reports);
 }
 async function outputEvalNotFound(id, evals) {
     if (isJsonMode()) {
@@ -895,7 +1030,7 @@ async function outputEvalNotFound(id, evals) {
             cliLogger.info("No evals found. Create an eval file in evals/.");
         }
     }
-    exitProcess(1);
+    return 1;
 }
 async function outputAgentNotFound(agentId) {
     if (isJsonMode()) {
@@ -908,7 +1043,20 @@ async function outputAgentNotFound(agentId) {
     else {
         cliLogger.error(`Agent "${agentId}" not found for eval target.`);
     }
-    exitProcess(1);
+    return 1;
+}
+async function outputToolNotFound(toolId) {
+    if (isJsonMode()) {
+        await outputJson(createErrorEnvelope("eval", {
+            code: "NOT_FOUND",
+            slug: "eval-tool-not-found",
+            message: `Tool "${toolId}" not found`,
+        }));
+    }
+    else {
+        cliLogger.error(`Tool "${toolId}" not found for eval target.`);
+    }
+    return 1;
 }
 async function outputEvalUsageError(message) {
     if (isJsonMode()) {
@@ -921,11 +1069,11 @@ async function outputEvalUsageError(message) {
     else {
         cliLogger.error(message);
     }
-    exitProcess(2);
+    return 2;
 }
 export async function evalCommand(options) {
     const projectDir = dntShim.Deno.cwd();
-    await withProjectSourceContext(projectDir, async ({ adapter, config }) => {
+    const exitCode = await withProjectSourceContext(projectDir, async ({ adapter, config }) => {
         await hydrateEvalRuntimeAuth(projectDir, config);
         const evalDiscovery = await discoverEvals({ projectDir, adapter, config });
         if (options.debug) {
@@ -937,17 +1085,17 @@ export async function evalCommand(options) {
             const evals = listEvals(evalDiscovery.evals, projectDir);
             if (isJsonMode()) {
                 await outputJson(createSuccessEnvelope("eval", { evals, errors: evalDiscovery.errors }));
-                return;
+                return undefined;
             }
             if (evals.length === 0) {
                 cliLogger.info("No evals found.");
-                return;
+                return undefined;
             }
             cliLogger.info("Evals:");
             for (const item of evals) {
                 cliLogger.info(`  - ${item.id} (${item.target})`);
             }
-            return;
+            return undefined;
         }
         if (!options.id) {
             if (isJsonMode()) {
@@ -960,101 +1108,125 @@ export async function evalCommand(options) {
             else {
                 cliLogger.error("Eval id is required. Usage: veryfront eval <eval-id>");
             }
-            exitProcess(2);
-            return;
+            return 2;
         }
         const evalId = normalizeEvalCliId(options.id);
         const evalItem = findEvalForCliId(evalDiscovery.evals, options.id);
         if (!evalItem) {
-            await outputEvalNotFound(evalId, evalDiscovery.evals);
-            return;
+            return await outputEvalNotFound(evalId, evalDiscovery.evals);
         }
         let modelComparisonConfig;
         try {
             modelComparisonConfig = await createResolvedEvalModelComparisonConfig(projectDir, options);
         }
         catch (error) {
-            await outputEvalUsageError(error instanceof Error ? error.message : String(error));
-            return;
+            return await outputEvalUsageError(error instanceof Error ? error.message : String(error));
         }
-        const discoveryConfig = createProjectDiscoveryConfig({
-            projectDir,
-            config,
-            fsAdapter: adapter.fs,
-            verbose: options.debug,
-        });
-        const projectDiscovery = await discoverAll(discoveryConfig);
-        const agentId = resolveAgentTargetId(evalItem.definition.target);
-        const agent = projectDiscovery.agents.get(agentId);
-        if (!agent) {
-            await outputAgentNotFound(agentId);
-            return;
+        if (evalItem.definition.targetKind !== "agent") {
+            if (modelComparisonConfig) {
+                return await outputEvalUsageError("Model comparison flags are only supported for agent evals.");
+            }
+            if (options.model || options.maxOutputTokens) {
+                return await outputEvalUsageError("--model and --max-output-tokens are only supported for agent evals.");
+            }
         }
-        if (modelComparisonConfig) {
-            await runEvalModelComparison({
-                evalItem,
-                agent,
-                options,
+        const selectedExporterIds = resolveEvalExporterIds(options);
+        const extensionSetup = await setupEvalCliExtensions(projectDir, config, selectedExporterIds);
+        try {
+            const discoveryConfig = createProjectDiscoveryConfig({
                 projectDir,
-                config: modelComparisonConfig.config,
-                policy: modelComparisonConfig.policy,
+                config,
+                fsAdapter: adapter.fs,
+                verbose: options.debug,
             });
-            return;
-        }
-        const runId = createEvalRunId();
-        const artifactPaths = createEvalArtifactPaths(options.reportDir ?? createDefaultEvalReportDir(runId, evalItem.id));
-        const provenance = await resolveEvalRunProvenance({
-            projectDir,
-            frameworkVersion: VERSION,
-        });
-        const billingGroupId = options.model
-            ? `${runId}_${sanitizeModelIdForPath(options.model)}`
-            : runId;
-        const exportConfig = createEvalCliExportConfig(evalItem, options, projectDir, artifactPaths);
-        const finalizedReport = await runEvalWithGatewayBillingGroup(billingGroupId, () => runEval(evalItem.definition, {
-            baseDir: options.datasetBase ?? projectDir,
-            runId,
-            adapters: {
-                agent: createAgentAdapter(agent, options),
-            },
-            metadata: {
-                provenance,
-                ...(options.model ? { model: options.model } : {}),
-            },
-        }));
-        const report = await exportEvalReportForCli(finalizedReport, exportConfig);
-        const baseline = options.baseline
-            ? compareEvalReports(report, await readEvalReport(options.baseline), createEvalBaselineComparisonPolicy(options))
-            : undefined;
-        await writeEvalArtifacts(report, artifactPaths, baseline);
-        if (options.report) {
-            await writeTextFileEnsuringDir(options.report, JSON.stringify(report, null, 2));
-        }
-        if (options.junit) {
-            await writeTextFileEnsuringDir(options.junit, createJunitXml(report));
-        }
-        if (options.writeBaseline) {
-            await writeTextFileEnsuringDir(options.writeBaseline, JSON.stringify(report, null, 2));
-        }
-        if (isJsonMode()) {
-            await outputJson(createSuccessEnvelope("eval", {
-                report,
-                summary: summarizeReportForCli(report),
-                baseline,
-                artifacts: artifactPaths,
+            const projectDiscovery = await discoverAll(discoveryConfig);
+            const agentId = evalItem.definition.targetKind === "agent"
+                ? resolveAgentTargetId(evalItem.definition.target)
+                : undefined;
+            const toolId = evalItem.definition.targetKind === "tool"
+                ? resolveToolTargetId(evalItem.definition.target)
+                : undefined;
+            const agent = agentId ? projectDiscovery.agents.get(agentId) : undefined;
+            const tool = toolId ? projectDiscovery.tools.get(toolId) : undefined;
+            if (agentId && !agent) {
+                return await outputAgentNotFound(agentId);
+            }
+            if (toolId && !tool) {
+                return await outputToolNotFound(toolId);
+            }
+            if (modelComparisonConfig) {
+                return await runEvalModelComparison({
+                    evalItem,
+                    agent: agent,
+                    options,
+                    projectDir,
+                    config: modelComparisonConfig.config,
+                    policy: modelComparisonConfig.policy,
+                    exporterRegistry: extensionSetup.exporterRegistry,
+                    projectReference: resolveEvalRuntimeProjectSlug(config),
+                });
+            }
+            const runId = createEvalRunId();
+            const artifactPaths = createEvalArtifactPaths(options.reportDir ?? createDefaultEvalReportDir(runId, evalItem.id));
+            const provenance = await resolveEvalRunProvenance({
+                projectDir,
+                frameworkVersion: VERSION,
+            });
+            const billingGroupId = options.model
+                ? `${runId}_${sanitizeModelIdForPath(options.model)}`
+                : runId;
+            const exportConfig = createEvalCliExportConfig(evalItem, options, projectDir, artifactPaths, extensionSetup.exporterRegistry, config);
+            const finalizedReport = await runEvalWithGatewayBillingGroup(billingGroupId, () => runEval(evalItem.definition, {
+                baseDir: options.datasetBase ?? projectDir,
+                runId,
+                adapters: evalItem.definition.targetKind === "tool"
+                    ? { tool: createToolAdapter(tool, createEvalToolExecutionContext(config)) }
+                    : { agent: createAgentAdapter(agent, options) },
+                metadata: {
+                    provenance,
+                    ...(options.model ? { model: options.model } : {}),
+                },
             }));
+            const report = await exportEvalReportForCli(finalizedReport, exportConfig);
+            const baseline = options.baseline
+                ? compareEvalReports(report, await readEvalReport(options.baseline), createEvalBaselineComparisonPolicy(options))
+                : undefined;
+            await writeEvalArtifacts(report, artifactPaths, baseline);
+            if (options.report) {
+                await writeTextFileEnsuringDir(options.report, JSON.stringify(report, null, 2));
+            }
+            if (options.junit) {
+                await writeTextFileEnsuringDir(options.junit, createJunitXml(report));
+            }
+            if (options.writeBaseline) {
+                await writeTextFileEnsuringDir(options.writeBaseline, JSON.stringify(report, null, 2));
+            }
+            if (isJsonMode()) {
+                await outputJson(createSuccessEnvelope("eval", {
+                    report,
+                    summary: summarizeReportForCli(report),
+                    baseline,
+                    artifacts: artifactPaths,
+                }));
+            }
+            else {
+                printReport(report, baseline);
+                cliLogger.info(`Report directory: ${artifactPaths.directory}`);
+                cliLogger.info(`Report markdown: ${artifactPaths.reportMarkdown}`);
+                if (options.report)
+                    cliLogger.info(`Report: ${options.report}`);
+                if (options.junit)
+                    cliLogger.info(`JUnit: ${options.junit}`);
+                if (options.writeBaseline)
+                    cliLogger.info(`Baseline written: ${options.writeBaseline}`);
+            }
+            return createEvalExitCode(report, baseline);
         }
-        else {
-            printReport(report, baseline);
-            cliLogger.info(`Report directory: ${artifactPaths.directory}`);
-            cliLogger.info(`Report markdown: ${artifactPaths.reportMarkdown}`);
-            if (options.report)
-                cliLogger.info(`Report: ${options.report}`);
-            if (options.junit)
-                cliLogger.info(`JUnit: ${options.junit}`);
-            if (options.writeBaseline)
-                cliLogger.info(`Baseline written: ${options.writeBaseline}`);
+        finally {
+            await extensionSetup.loader.teardownAll();
         }
-        exitProcess(createEvalExitCode(report, baseline));
     });
+    if (typeof exitCode === "number") {
+        exitProcess(exitCode);
+    }
 }