veryfront 0.1.871 → 0.1.873

package/esm/src/eval/expect.js

@@ -0,0 +1,62 @@
+import { getOutputText } from "./metrics.js";
+function createExpectation(checks, base) {
+    function record(severity, threshold) {
+        const check = {
+            ...base,
+            severity,
+            ...(threshold ? { evidence: { ...(base.evidence ?? {}), threshold } } : {}),
+        };
+        checks.push(check);
+        return check;
+    }
+    return {
+        gate: (threshold) => record("gate", threshold),
+        soft: (threshold) => record("soft", threshold),
+        budget: (threshold) => record("budget", threshold),
+    };
+}
+export function createEvalExpect(record, checks) {
+    return {
+        completed() {
+            return createExpectation(checks, {
+                name: "expect.completed",
+                family: "check",
+                score: record.completed ? 1 : 0,
+                pass: record.completed,
+                ...(record.error ? { explanation: record.error } : {}),
+            });
+        },
+        outputContains(text) {
+            const output = getOutputText(record.output);
+            const pass = output.includes(text);
+            return createExpectation(checks, {
+                name: "expect.outputContains",
+                family: "check",
+                score: pass ? 1 : 0,
+                pass,
+                evidence: { text },
+            });
+        },
+        noFailedTools() {
+            const failedTools = record.trace.toolCalls
+                .filter((tool) => tool.status === "error" || typeof tool.error === "string")
+                .map((tool) => tool.name);
+            return createExpectation(checks, {
+                name: "expect.noFailedTools",
+                family: "check",
+                score: failedTools.length === 0 ? 1 : 0,
+                pass: failedTools.length === 0,
+                ...(failedTools.length > 0 ? { evidence: { failedTools } } : {}),
+            });
+        },
+    };
+}
+export function createEvalCheckContext(input) {
+    return {
+        definition: input.definition,
+        example: input.example,
+        repetition: input.repetition,
+        record: input.record,
+        expect: createEvalExpect(input.record, input.checks),
+    };
+}

package/esm/src/eval/factory.d.ts

@@ -0,0 +1,6 @@
+import type { EvalAgentInput, EvalDefinition } from "./types.js";
+/** Define a V1 eval that targets a Veryfront agent. */
+export declare function evalAgent(input: EvalAgentInput): EvalDefinition;
+/** Check whether a value is a normalized eval definition. */
+export declare function isEvalDefinition(value: unknown): value is EvalDefinition;
+//# sourceMappingURL=factory.d.ts.map

package/esm/src/eval/factory.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../../src/src/eval/factory.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAe,cAAc,EAAoB,MAAM,YAAY,CAAC;AAwBhG,uDAAuD;AACvD,wBAAgB,SAAS,CAAC,KAAK,EAAE,cAAc,GAAG,cAAc,CAmB/D;AAED,6DAA6D;AAC7D,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,cAAc,CASxE"}

package/esm/src/eval/factory.js

@@ -0,0 +1,50 @@
+import { datasets } from "./datasets.js";
+import { createEvalValidationError } from "./validation.js";
+function isEvalDataset(value) {
+    return !!value &&
+        typeof value === "object" &&
+        typeof value.kind === "string" &&
+        typeof value.load === "function";
+}
+function normalizeDataset(dataset) {
+    return isEvalDataset(dataset) ? dataset : datasets.inline(dataset);
+}
+function normalizeRepetitions(repetitions) {
+    const value = repetitions ?? 1;
+    if (!Number.isInteger(value) || value < 1) {
+        throw createEvalValidationError("Eval repetitions must be an integer greater than or equal to 1");
+    }
+    return value;
+}
+/** Define a V1 eval that targets a Veryfront agent. */
+export function evalAgent(input) {
+    if (typeof input.target !== "string" || input.target.trim() === "") {
+        throw createEvalValidationError("Eval target must be a non-empty string");
+    }
+    return {
+        kind: "eval",
+        targetKind: "agent",
+        id: input.id ?? "",
+        name: input.name ?? input.id ?? input.target,
+        ...(input.description ? { description: input.description } : {}),
+        target: input.target,
+        dataset: normalizeDataset(input.dataset),
+        metrics: input.metrics ?? [],
+        repetitions: normalizeRepetitions(input.repetitions),
+        tags: input.tags ?? [],
+        metadata: input.metadata ?? {},
+        ...(input.check ? { check: input.check } : {}),
+    };
+}
+/** Check whether a value is a normalized eval definition. */
+export function isEvalDefinition(value) {
+    if (!value || typeof value !== "object")
+        return false;
+    const candidate = value;
+    return candidate.kind === "eval" &&
+        candidate.targetKind === "agent" &&
+        typeof candidate.target === "string" &&
+        isEvalDataset(candidate.dataset) &&
+        Array.isArray(candidate.metrics) &&
+        typeof candidate.repetitions === "number";
+}

package/esm/src/eval/index.d.ts

@@ -0,0 +1,33 @@
+/**
+ * First-class eval primitives for agent quality checks.
+ *
+ * @module eval
+ *
+ * @example
+ * ```ts
+ * import { datasets, evalAgent, metrics } from "veryfront/eval";
+ *
+ * export default evalAgent({
+ *   target: "agent:researcher",
+ *   dataset: datasets.inline([
+ *     { id: "q1", input: "Capital of France?", reference: "Paris" },
+ *   ]),
+ *   metrics: [
+ *     metrics.answer.contains({ text: "Paris" }).gate(),
+ *     metrics.agent.noFailedTools().gate(),
+ *   ],
+ * });
+ * ```
+ */
+import "../../_dnt.polyfills.js";
+export { datasets } from "./datasets.js";
+export { evalAgent, isEvalDefinition } from "./factory.js";
+export { metrics } from "./metrics.js";
+export { createEvalReport, summarizeEvalRecords } from "./report.js";
+export { runEval } from "./runner.js";
+export { deriveEvalId, discoverEvals, findEvalById } from "./discovery.js";
+export { createEvalSourceDocument, getEvalEditableFieldSchema, getEvalRunSchema, getEvalSourceDocumentSchema, getEvalSourcePatchSchema, getEvalSourceReferenceSchema, getEvalStudioCapabilitySchema, } from "./studio.js";
+export type { DiscoveredEval, EvalDiscoveryOptions, EvalDiscoveryResult } from "./discovery.js";
+export type { EvalAgentAdapter, EvalAgentAdapterContext, EvalAgentAdapterResult, EvalAgentInput, EvalCheckContext, EvalDataset, EvalDatasetLoadContext, EvalDefinition, EvalExample, EvalExampleInput, EvalExpect, EvalExpectation, EvalMetric, EvalMetricContext, EvalMetricFamily, EvalMetricResult, EvalMetricSummary, EvalMetricThreshold, EvalRecord, EvalReport, EvalReportSummary, EvalSeverity, EvalSource, EvalTargetKind, EvalToolCall, EvalTrace, EvalUsage, RunEvalOptions, } from "./types.js";
+export type { CreateEvalSourceDocumentOptions, EvalEditableField, EvalRun, EvalSourceDocument, EvalSourcePatch, EvalSourceReference, EvalStudioCapability, } from "./studio.js";
+//# sourceMappingURL=index.d.ts.map

package/esm/src/eval/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EACL,wBAAwB,EACxB,0BAA0B,EAC1B,gBAAgB,EAChB,2BAA2B,EAC3B,wBAAwB,EACxB,4BAA4B,EAC5B,6BAA6B,GAC9B,MAAM,aAAa,CAAC;AAErB,YAAY,EAAE,cAAc,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAChG,YAAY,EACV,gBAAgB,EAChB,uBAAuB,EACvB,sBAAsB,EACtB,cAAc,EACd,gBAAgB,EAChB,WAAW,EACX,sBAAsB,EACtB,cAAc,EACd,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,eAAe,EACf,UAAU,EACV,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,UAAU,EACV,UAAU,EACV,iBAAiB,EACjB,YAAY,EACZ,UAAU,EACV,cAAc,EACd,YAAY,EACZ,SAAS,EACT,SAAS,EACT,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,+BAA+B,EAC/B,iBAAiB,EACjB,OAAO,EACP,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,aAAa,CAAC"}

package/esm/src/eval/index.js

@@ -0,0 +1,29 @@
+/**
+ * First-class eval primitives for agent quality checks.
+ *
+ * @module eval
+ *
+ * @example
+ * ```ts
+ * import { datasets, evalAgent, metrics } from "veryfront/eval";
+ *
+ * export default evalAgent({
+ *   target: "agent:researcher",
+ *   dataset: datasets.inline([
+ *     { id: "q1", input: "Capital of France?", reference: "Paris" },
+ *   ]),
+ *   metrics: [
+ *     metrics.answer.contains({ text: "Paris" }).gate(),
+ *     metrics.agent.noFailedTools().gate(),
+ *   ],
+ * });
+ * ```
+ */
+import "../../_dnt.polyfills.js";
+export { datasets } from "./datasets.js";
+export { evalAgent, isEvalDefinition } from "./factory.js";
+export { metrics } from "./metrics.js";
+export { createEvalReport, summarizeEvalRecords } from "./report.js";
+export { runEval } from "./runner.js";
+export { deriveEvalId, discoverEvals, findEvalById } from "./discovery.js";
+export { createEvalSourceDocument, getEvalEditableFieldSchema, getEvalRunSchema, getEvalSourceDocumentSchema, getEvalSourcePatchSchema, getEvalSourceReferenceSchema, getEvalStudioCapabilitySchema, } from "./studio.js";

package/esm/src/eval/metrics.d.ts

@@ -0,0 +1,55 @@
+import type { EvalMetric } from "./types.js";
+type JudgeRubricInput = {
+    rubric: string;
+    judge?: (input: {
+        rubric: string;
+        input: unknown;
+        output: Record<string, unknown>;
+        reference?: unknown;
+        metadata: Record<string, unknown>;
+    }) => Promise<{
+        score: number;
+        pass?: boolean;
+        explanation?: string;
+    }>;
+};
+declare function getOutputText(output: unknown): string;
+declare function stableStringify(value: unknown): string;
+/** Metric factories for deterministic answers, agent behavior, operations, and judges. */
+export declare const metrics: {
+    readonly answer: {
+        readonly exactMatch: () => EvalMetric;
+        readonly contains: (options: {
+            text: string;
+            caseSensitive?: boolean;
+        }) => EvalMetric;
+        readonly regex: (options: {
+            pattern: string;
+            flags?: string;
+        }) => EvalMetric;
+        readonly jsonMatch: (options: {
+            expected?: unknown;
+        }) => EvalMetric;
+    };
+    readonly agent: {
+        readonly noFailedTools: () => EvalMetric;
+    };
+    readonly ops: {
+        readonly latency: (options: {
+            maxMs: number;
+        }) => EvalMetric;
+        readonly tokens: (options: {
+            maxTotal?: number;
+            maxInput?: number;
+            maxOutput?: number;
+        }) => EvalMetric;
+        readonly cost: (options: {
+            maxUsd: number;
+        }) => EvalMetric;
+    };
+    readonly judge: {
+        readonly rubric: (options: JudgeRubricInput) => EvalMetric;
+    };
+};
+export { getOutputText, stableStringify };
+//# sourceMappingURL=metrics.d.ts.map

package/esm/src/eval/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../src/src/eval/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EAMX,MAAM,YAAY,CAAC;AAIpB,KAAK,gBAAgB,GAAG;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE;QACd,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,OAAO,CAAC;QACf,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC,KAAK,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACxE,CAAC;AAEF,iBAAS,aAAa,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,CAQ9C;AASD,iBAAS,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAE/C;AA2FD,0FAA0F;AAC1F,eAAO,MAAM,OAAO;;mCAEF,UAAU;qCAON;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,aAAa,CAAC,EAAE,OAAO,CAAA;SAAE,KAAG,UAAU;kCAWzD;YAAE,OAAO,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,KAAG,UAAU;sCAY5C;YAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;SAAE,KAAG,UAAU;;;sCAWrC,UAAU;;;oCAgBV;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,KAAG,UAAU;mCAc/B;YAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;YAAC,SAAS,CAAC,EAAE,MAAM,CAAA;SAAE,KAAG,UAAU;iCAoB3E;YAAE,MAAM,EAAE,MAAM,CAAA;SAAE,KAAG,UAAU;;;mCAiB7B,gBAAgB,KAAG,UAAU;;CAmCvC,CAAC;AAEX,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,CAAC"}

package/esm/src/eval/metrics.js

@@ -0,0 +1,223 @@
+function getOutputText(output) {
+    if (typeof output === "string")
+        return output;
+    if (output && typeof output === "object") {
+        const record = output;
+        if (typeof record.text === "string")
+            return record.text;
+        if (typeof record.output === "string")
+            return record.output;
+    }
+    return stableStringify(output);
+}
+function getOutputJson(output) {
+    if (output && typeof output === "object" && "json" in output) {
+        return output.json;
+    }
+    return output;
+}
+function stableStringify(value) {
+    return JSON.stringify(sortJsonValue(value));
+}
+function sortJsonValue(value) {
+    if (Array.isArray(value))
+        return value.map(sortJsonValue);
+    if (!value || typeof value !== "object")
+        return value;
+    const sorted = {};
+    for (const key of Object.keys(value).sort()) {
+        sorted[key] = sortJsonValue(value[key]);
+    }
+    return sorted;
+}
+function withSeverity(metric, severity, threshold) {
+    const base = { ...metric, severity, ...(threshold ? { threshold } : {}) };
+    return {
+        ...base,
+        async evaluate(record, context) {
+            const result = await metric.evaluate(record, context);
+            const next = {
+                ...result,
+                name: metric.name,
+                family: metric.family,
+                severity,
+            };
+            if (!threshold || result.skipped || typeof next.score !== "number") {
+                return next;
+            }
+            const minPass = threshold.min === undefined || next.score >= threshold.min;
+            const maxPass = threshold.max === undefined || next.score <= threshold.max;
+            const thresholdPass = minPass && maxPass;
+            return {
+                ...next,
+                pass: next.pass === undefined ? thresholdPass : next.pass && thresholdPass,
+            };
+        },
+        gate(nextThreshold) {
+            return withSeverity(base, "gate", nextThreshold ?? threshold);
+        },
+        soft(nextThreshold) {
+            return withSeverity(base, "soft", nextThreshold ?? threshold);
+        },
+        budget(nextThreshold) {
+            return withSeverity(base, "budget", nextThreshold ?? threshold);
+        },
+    };
+}
+function createMetric(name, family, evaluator, config) {
+    const metric = {
+        name,
+        family,
+        severity: "gate",
+        ...(config ? { config } : {}),
+        async evaluate(record) {
+            const result = await evaluator(record);
+            return {
+                ...result,
+                name,
+                family,
+                severity: "gate",
+            };
+        },
+    };
+    return withSeverity(metric, "gate");
+}
+function scoreResult(name, family, severity, pass, score = pass ? 1 : 0) {
+    return { name, family, severity, score, pass };
+}
+function isToolFailed(toolCall) {
+    return toolCall.status === "error" || typeof toolCall.error === "string";
+}
+/** Metric factories for deterministic answers, agent behavior, operations, and judges. */
+export const metrics = {
+    answer: {
+        exactMatch() {
+            return createMetric("answer.exactMatch", "answer", (record) => {
+                const pass = getOutputText(record.output) === getOutputText(record.reference);
+                return scoreResult("answer.exactMatch", "answer", "gate", pass);
+            });
+        },
+        contains(options) {
+            return createMetric("answer.contains", "answer", (record) => {
+                const actual = getOutputText(record.output);
+                const expected = options.text;
+                const pass = options.caseSensitive
+                    ? actual.includes(expected)
+                    : actual.toLowerCase().includes(expected.toLowerCase());
+                return scoreResult("answer.contains", "answer", "gate", pass);
+            }, options);
+        },
+        regex(options) {
+            return createMetric("answer.regex", "answer", (record) => {
+                const pattern = new RegExp(options.pattern, options.flags);
+                return scoreResult("answer.regex", "answer", "gate", pattern.test(getOutputText(record.output)));
+            }, options);
+        },
+        jsonMatch(options) {
+            return createMetric("answer.jsonMatch", "answer", (record) => {
+                const expected = Object.hasOwn(options, "expected") ? options.expected : record.reference;
+                const actual = getOutputJson(record.output);
+                const pass = stableStringify(actual) === stableStringify(expected);
+                return scoreResult("answer.jsonMatch", "answer", "gate", pass);
+            }, options);
+        },
+    },
+    agent: {
+        noFailedTools() {
+            return createMetric("agent.noFailedTools", "agent", (record) => {
+                const failedTools = record.trace.toolCalls.filter(isToolFailed).map((tool) => tool.name);
+                return {
+                    name: "agent.noFailedTools",
+                    family: "agent",
+                    severity: "gate",
+                    score: failedTools.length === 0 ? 1 : 0,
+                    pass: failedTools.length === 0,
+                    ...(failedTools.length > 0 ? { evidence: { failedTools } } : {}),
+                };
+            });
+        },
+    },
+    ops: {
+        latency(options) {
+            return createMetric("ops.latency", "ops", (record) => {
+                const pass = record.durationMs <= options.maxMs;
+                return {
+                    name: "ops.latency",
+                    family: "ops",
+                    severity: "budget",
+                    score: pass ? 1 : 0,
+                    pass,
+                    evidence: { durationMs: record.durationMs, maxMs: options.maxMs },
+                };
+            }, options);
+        },
+        tokens(options) {
+            return createMetric("ops.tokens", "ops", (record) => {
+                const inputOk = options.maxInput === undefined ||
+                    (record.usage.inputTokens ?? 0) <= options.maxInput;
+                const outputOk = options.maxOutput === undefined ||
+                    (record.usage.outputTokens ?? 0) <= options.maxOutput;
+                const totalOk = options.maxTotal === undefined ||
+                    (record.usage.totalTokens ?? 0) <= options.maxTotal;
+                const pass = inputOk && outputOk && totalOk;
+                return {
+                    name: "ops.tokens",
+                    family: "ops",
+                    severity: "budget",
+                    score: pass ? 1 : 0,
+                    pass,
+                    evidence: { usage: record.usage, limits: options },
+                };
+            }, options);
+        },
+        cost(options) {
+            return createMetric("ops.cost", "ops", (record) => {
+                const costUsd = record.usage.costUsd ?? 0;
+                const pass = costUsd <= options.maxUsd;
+                return {
+                    name: "ops.cost",
+                    family: "ops",
+                    severity: "budget",
+                    score: pass ? 1 : 0,
+                    pass,
+                    evidence: { costUsd, maxUsd: options.maxUsd },
+                };
+            }, options);
+        },
+    },
+    judge: {
+        rubric(options) {
+            return createMetric("judge.rubric", "judge", async (record) => {
+                if (!options.judge) {
+                    return {
+                        name: "judge.rubric",
+                        family: "judge",
+                        severity: "gate",
+                        skipped: true,
+                        explanation: "No judge function was provided.",
+                    };
+                }
+                const output = record.output && typeof record.output === "object"
+                    ? record.output
+                    : { text: getOutputText(record.output) };
+                const judged = await options.judge({
+                    rubric: options.rubric,
+                    input: record.input,
+                    output,
+                    reference: record.reference,
+                    metadata: record.metadata,
+                });
+                const min = 0;
+                return {
+                    name: "judge.rubric",
+                    family: "judge",
+                    severity: "gate",
+                    score: judged.score,
+                    pass: judged.pass ?? judged.score > min,
+                    ...(judged.explanation ? { explanation: judged.explanation } : {}),
+                };
+            }, { rubric: options.rubric });
+        },
+    },
+};
+export { getOutputText, stableStringify };

package/esm/src/eval/report.d.ts

@@ -0,0 +1,12 @@
+import type { EvalDefinition, EvalRecord, EvalReport, EvalReportSummary } from "./types.js";
+/** Summarize eval records into pass/fail and metric aggregates. */
+export declare function summarizeEvalRecords(records: EvalRecord[]): EvalReportSummary;
+/** Create a JSON-serializable eval report from executed records. */
+export declare function createEvalReport(input: {
+    definition: EvalDefinition;
+    records: EvalRecord[];
+    runId: string;
+    startedAt: Date;
+    endedAt: Date;
+}): EvalReport;
+//# sourceMappingURL=report.d.ts.map

package/esm/src/eval/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../../src/src/eval/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EAGd,UAAU,EACV,UAAU,EACV,iBAAiB,EAClB,MAAM,YAAY,CAAC;AA6CpB,mEAAmE;AACnE,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,iBAAiB,CAU7E;AAED,oEAAoE;AACpE,wBAAgB,gBAAgB,CAAC,KAAK,EAAE;IACtC,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,IAAI,CAAC;IAChB,OAAO,EAAE,IAAI,CAAC;CACf,GAAG,UAAU,CAYb"}

package/esm/src/eval/report.js

@@ -0,0 +1,64 @@
+function isBlockingFailure(result) {
+    return !result.skipped && result.pass === false &&
+        (result.severity === "gate" || result.severity === "budget");
+}
+function recordPassed(record) {
+    const results = [...(record.metrics ?? []), ...(record.checks ?? [])];
+    return results.every((result) => !isBlockingFailure(result));
+}
+function summarizeMetricResults(records) {
+    const summaries = new Map();
+    for (const record of records) {
+        for (const result of [...(record.metrics ?? []), ...(record.checks ?? [])]) {
+            const key = `${result.name}:${result.family}:${result.severity}`;
+            const summary = summaries.get(key) ?? {
+                name: result.name,
+                family: result.family,
+                severity: result.severity,
+                passed: 0,
+                failed: 0,
+                skipped: 0,
+                passRate: 0,
+            };
+            if (result.skipped) {
+                summary.skipped += 1;
+            }
+            else if (result.pass === false) {
+                summary.failed += 1;
+            }
+            else {
+                summary.passed += 1;
+            }
+            const denominator = summary.passed + summary.failed;
+            summary.passRate = denominator === 0 ? 1 : summary.passed / denominator;
+            summaries.set(key, summary);
+        }
+    }
+    return [...summaries.values()];
+}
+/** Summarize eval records into pass/fail and metric aggregates. */
+export function summarizeEvalRecords(records) {
+    const passed = records.filter(recordPassed).length;
+    const failed = records.length - passed;
+    return {
+        records: records.length,
+        passed,
+        failed,
+        passRate: records.length === 0 ? 1 : passed / records.length,
+        metrics: summarizeMetricResults(records),
+    };
+}
+/** Create a JSON-serializable eval report from executed records. */
+export function createEvalReport(input) {
+    return {
+        kind: "eval-report",
+        runId: input.runId,
+        definitionId: input.definition.id,
+        targetKind: input.definition.targetKind,
+        target: input.definition.target,
+        startedAt: input.startedAt.toISOString(),
+        endedAt: input.endedAt.toISOString(),
+        summary: summarizeEvalRecords(input.records),
+        records: input.records,
+    };
+}

package/esm/src/eval/runner.d.ts

@@ -0,0 +1,4 @@
+import type { EvalDefinition, RunEvalOptions } from "./types.js";
+/** Execute an eval locally with injected target adapters. */
+export declare function runEval(definition: EvalDefinition, options: RunEvalOptions): Promise<import("./types.js").EvalReport>;
+//# sourceMappingURL=runner.d.ts.map

package/esm/src/eval/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/src/eval/runner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAEV,cAAc,EAKd,cAAc,EACf,MAAM,YAAY,CAAC;AAkGpB,6DAA6D;AAC7D,wBAAsB,OAAO,CAC3B,UAAU,EAAE,cAAc,EAC1B,OAAO,EAAE,cAAc,4CAqBxB"}

package/esm/src/eval/runner.js

@@ -0,0 +1,101 @@
+import * as dntShim from "../../_dnt.shims.js";
+import { createEvalCheckContext } from "./expect.js";
+import { createEvalReport } from "./report.js";
+function createRunId(now) {
+    return `evalrun_${now.toISOString().replace(/[-:.]/g, "").replace("T", "_").replace("Z", "")}`;
+}
+function normalizeTrace(trace) {
+    return {
+        events: trace?.events ?? [],
+        toolCalls: trace?.toolCalls ?? [],
+    };
+}
+function normalizeUsage(usage) {
+    return usage ?? {};
+}
+function normalizeAdapterResult(result) {
+    return typeof result === "string" ? { text: result } : result;
+}
+function normalizeOutput(result) {
+    if (Object.hasOwn(result, "output"))
+        return result.output;
+    if (Object.hasOwn(result, "json"))
+        return { json: result.json };
+    if (Object.hasOwn(result, "text"))
+        return { text: result.text };
+    return result;
+}
+function isBlockingFailure(record) {
+    return [...(record.metrics ?? []), ...(record.checks ?? [])].some((result) => !result.skipped && result.pass === false &&
+        (result.severity === "gate" || result.severity === "budget"));
+}
+async function runRecord(definition, options, example, repetition) {
+    const started = Date.now();
+    let result;
+    try {
+        result = normalizeAdapterResult(await options.adapters.agent({ definition, example, repetition }));
+    }
+    catch (error) {
+        result = {
+            text: "",
+            completed: false,
+            error: error instanceof Error ? error.message : String(error),
+        };
+    }
+    const output = normalizeOutput(result);
+    const record = {
+        id: `${example.id}:${repetition}`,
+        evalId: definition.id,
+        exampleId: example.id,
+        repetition,
+        input: example.input,
+        output,
+        ...(Object.hasOwn(example, "reference") ? { reference: example.reference } : {}),
+        metadata: example.metadata ?? {},
+        trace: normalizeTrace(result.trace),
+        usage: normalizeUsage(result.usage),
+        durationMs: result.durationMs ?? Date.now() - started,
+        completed: result.completed ?? !result.error,
+        ...(result.error ? { error: result.error } : {}),
+    };
+    const metricResults = [];
+    for (const metric of definition.metrics) {
+        metricResults.push(await metric.evaluate(record));
+    }
+    record.metrics = metricResults;
+    const checks = [];
+    if (definition.check) {
+        await definition.check(createEvalCheckContext({
+            definition,
+            example,
+            repetition,
+            record,
+            checks,
+        }));
+    }
+    record.checks = checks;
+    if (isBlockingFailure(record)) {
+        record.completed = record.completed && true;
+    }
+    return record;
+}
+/** Execute an eval locally with injected target adapters. */
+export async function runEval(definition, options) {
+    const startedAt = options.now?.() ?? new Date();
+    const baseDir = options.baseDir ?? dntShim.Deno.cwd();
+    const examples = await definition.dataset.load({ baseDir });
+    const records = [];
+    for (const example of examples) {
+        for (let repetition = 1; repetition <= definition.repetitions; repetition += 1) {
+            records.push(await runRecord(definition, options, example, repetition));
+        }
+    }
+    const endedAt = options.now?.() ?? new Date();
+    return createEvalReport({
+        definition,
+        records,
+        runId: options.runId ?? createRunId(startedAt),
+        startedAt,
+        endedAt,
+    });
+}