veryfront 0.1.871 → 0.1.873

package/esm/src/eval/studio.d.ts

@@ -0,0 +1,177 @@
+import type { InferSchema } from "../extensions/schema/index.js";
+import type { DiscoveredEval } from "./discovery.js";
+/** Schema for Eval Studio capabilities. */
+export declare const getEvalStudioCapabilitySchema: () => import("../internal-agents/schema.js").Schema<"project.evals.read" | "project.evals.write">;
+/** Schema for an editable Eval source field name. */
+export declare const getEvalEditableFieldSchema: () => import("../internal-agents/schema.js").Schema<"target" | "description" | "metrics" | "name" | "metadata" | "tags" | "check" | "dataset" | "repetitions">;
+/** Schema for an Eval source reference. */
+export declare const getEvalSourceReferenceSchema: () => import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+    filePath: import("../internal-agents/schema.js").Schema<string>;
+    exportName: import("../internal-agents/schema.js").Schema<string>;
+    content: import("../internal-agents/schema.js").Schema<string | undefined>;
+}>>;
+/** Schema for an Eval example in Studio source documents. */
+export declare const getEvalSourceExampleSchema: () => import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+    id: import("../internal-agents/schema.js").Schema<string>;
+    input: import("../internal-agents/schema.js").Schema<unknown>;
+    reference: import("../internal-agents/schema.js").Schema<unknown>;
+    metadata: import("../internal-agents/schema.js").Schema<Record<string, unknown> | undefined>;
+}>>;
+/** Schema for an Eval dataset in Studio source documents. */
+export declare const getEvalSourceDatasetSchema: () => import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+    kind: import("../internal-agents/schema.js").Schema<"json" | "dynamic" | "inline" | "jsonl">;
+    path: import("../internal-agents/schema.js").Schema<string | undefined>;
+    examples: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+        id: import("../internal-agents/schema.js").Schema<string>;
+        input: import("../internal-agents/schema.js").Schema<unknown>;
+        reference: import("../internal-agents/schema.js").Schema<unknown>;
+        metadata: import("../internal-agents/schema.js").Schema<Record<string, unknown> | undefined>;
+    }>[] | undefined>;
+    editable: import("../internal-agents/schema.js").Schema<boolean>;
+    dynamic: import("../internal-agents/schema.js").Schema<boolean>;
+}>>;
+/** Schema for an Eval metric in Studio source documents. */
+export declare const getEvalSourceMetricSchema: () => import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+    name: import("../internal-agents/schema.js").Schema<string>;
+    family: import("../internal-agents/schema.js").Schema<"agent" | "check" | "answer" | "ops" | "judge">;
+    severity: import("../internal-agents/schema.js").Schema<"gate" | "soft" | "budget">;
+    threshold: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+        min: import("../internal-agents/schema.js").Schema<number | undefined>;
+        max: import("../internal-agents/schema.js").Schema<number | undefined>;
+    }> | undefined>;
+    config: import("../internal-agents/schema.js").Schema<Record<string, unknown> | undefined>;
+    editable: import("../internal-agents/schema.js").Schema<boolean>;
+    dynamic: import("../internal-agents/schema.js").Schema<boolean>;
+}>>;
+/** Schema for a Studio-editable Eval source document. */
+export declare const getEvalSourceDocumentSchema: () => import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+    kind: import("../internal-agents/schema.js").Schema<"eval-source-document">;
+    id: import("../internal-agents/schema.js").Schema<string>;
+    name: import("../internal-agents/schema.js").Schema<string>;
+    description: import("../internal-agents/schema.js").Schema<string | undefined>;
+    targetKind: import("../internal-agents/schema.js").Schema<"agent">;
+    target: import("../internal-agents/schema.js").Schema<string>;
+    source: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+        filePath: import("../internal-agents/schema.js").Schema<string>;
+        exportName: import("../internal-agents/schema.js").Schema<string>;
+        content: import("../internal-agents/schema.js").Schema<string | undefined>;
+    }>>;
+    dataset: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+        kind: import("../internal-agents/schema.js").Schema<"json" | "dynamic" | "inline" | "jsonl">;
+        path: import("../internal-agents/schema.js").Schema<string | undefined>;
+        examples: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+            id: import("../internal-agents/schema.js").Schema<string>;
+            input: import("../internal-agents/schema.js").Schema<unknown>;
+            reference: import("../internal-agents/schema.js").Schema<unknown>;
+            metadata: import("../internal-agents/schema.js").Schema<Record<string, unknown> | undefined>;
+        }>[] | undefined>;
+        editable: import("../internal-agents/schema.js").Schema<boolean>;
+        dynamic: import("../internal-agents/schema.js").Schema<boolean>;
+    }>>;
+    metrics: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+        name: import("../internal-agents/schema.js").Schema<string>;
+        family: import("../internal-agents/schema.js").Schema<"agent" | "check" | "answer" | "ops" | "judge">;
+        severity: import("../internal-agents/schema.js").Schema<"gate" | "soft" | "budget">;
+        threshold: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+            min: import("../internal-agents/schema.js").Schema<number | undefined>;
+            max: import("../internal-agents/schema.js").Schema<number | undefined>;
+        }> | undefined>;
+        config: import("../internal-agents/schema.js").Schema<Record<string, unknown> | undefined>;
+        editable: import("../internal-agents/schema.js").Schema<boolean>;
+        dynamic: import("../internal-agents/schema.js").Schema<boolean>;
+    }>[]>;
+    repetitions: import("../internal-agents/schema.js").Schema<number>;
+    tags: import("../internal-agents/schema.js").Schema<string[]>;
+    metadata: import("../internal-agents/schema.js").Schema<Record<string, unknown>>;
+    editableFields: import("../internal-agents/schema.js").Schema<("target" | "description" | "metrics" | "name" | "metadata" | "tags" | "check" | "dataset" | "repetitions")[]>;
+    dynamicFields: import("../internal-agents/schema.js").Schema<("target" | "description" | "metrics" | "name" | "metadata" | "tags" | "check" | "dataset" | "repetitions")[]>;
+    capabilities: import("../internal-agents/schema.js").Schema<("project.evals.read" | "project.evals.write")[]>;
+}>>;
+/** Schema for a source patch submitted from an Eval editor. */
+export declare const getEvalSourcePatchSchema: () => import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+    kind: import("../internal-agents/schema.js").Schema<"eval-source-patch">;
+    id: import("../internal-agents/schema.js").Schema<string>;
+    source: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+        filePath: import("../internal-agents/schema.js").Schema<string>;
+        exportName: import("../internal-agents/schema.js").Schema<string>;
+        content: import("../internal-agents/schema.js").Schema<string | undefined>;
+    }>>;
+    fields: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+        name: import("../internal-agents/schema.js").Schema<string | undefined>;
+        description: import("../internal-agents/schema.js").Schema<string | undefined>;
+        target: import("../internal-agents/schema.js").Schema<string | undefined>;
+        dataset: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+            kind: import("../internal-agents/schema.js").Schema<"json" | "dynamic" | "inline" | "jsonl">;
+            path: import("../internal-agents/schema.js").Schema<string | undefined>;
+            examples: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+                id: import("../internal-agents/schema.js").Schema<string>;
+                input: import("../internal-agents/schema.js").Schema<unknown>;
+                reference: import("../internal-agents/schema.js").Schema<unknown>;
+                metadata: import("../internal-agents/schema.js").Schema<Record<string, unknown> | undefined>;
+            }>[] | undefined>;
+            editable: import("../internal-agents/schema.js").Schema<boolean>;
+            dynamic: import("../internal-agents/schema.js").Schema<boolean>;
+        }> | undefined>;
+        repetitions: import("../internal-agents/schema.js").Schema<number | undefined>;
+        tags: import("../internal-agents/schema.js").Schema<string[] | undefined>;
+        metadata: import("../internal-agents/schema.js").Schema<Record<string, unknown> | undefined>;
+        metrics: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+            name: import("../internal-agents/schema.js").Schema<string>;
+            family: import("../internal-agents/schema.js").Schema<"agent" | "check" | "answer" | "ops" | "judge">;
+            severity: import("../internal-agents/schema.js").Schema<"gate" | "soft" | "budget">;
+            threshold: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+                min: import("../internal-agents/schema.js").Schema<number | undefined>;
+                max: import("../internal-agents/schema.js").Schema<number | undefined>;
+            }> | undefined>;
+            config: import("../internal-agents/schema.js").Schema<Record<string, unknown> | undefined>;
+            editable: import("../internal-agents/schema.js").Schema<boolean>;
+            dynamic: import("../internal-agents/schema.js").Schema<boolean>;
+        }>[] | undefined>;
+    }>>;
+}>>;
+/** Schema for V2-ready Eval run projections. */
+export declare const getEvalRunSchema: () => import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+    kind: import("../internal-agents/schema.js").Schema<"eval-run">;
+    runId: import("../internal-agents/schema.js").Schema<string>;
+    evalId: import("../internal-agents/schema.js").Schema<string>;
+    status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
+    targetKind: import("../internal-agents/schema.js").Schema<"agent">;
+    target: import("../internal-agents/schema.js").Schema<string>;
+    source: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+        filePath: import("../internal-agents/schema.js").Schema<string>;
+        exportName: import("../internal-agents/schema.js").Schema<string>;
+        content: import("../internal-agents/schema.js").Schema<string | undefined>;
+    }> | undefined>;
+    summary: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
+        records: import("../internal-agents/schema.js").Schema<number>;
+        passed: import("../internal-agents/schema.js").Schema<number>;
+        failed: import("../internal-agents/schema.js").Schema<number>;
+        passRate: import("../internal-agents/schema.js").Schema<number>;
+    }> | null>;
+    reportPath: import("../internal-agents/schema.js").Schema<string | null>;
+    error: import("../internal-agents/schema.js").Schema<unknown>;
+    metadata: import("../internal-agents/schema.js").Schema<Record<string, unknown>>;
+    createdAt: import("../internal-agents/schema.js").Schema<string>;
+    startedAt: import("../internal-agents/schema.js").Schema<string | null>;
+    completedAt: import("../internal-agents/schema.js").Schema<string | null>;
+}>>;
+/** Capability string Studio uses for Eval read and write access. */
+export type EvalStudioCapability = InferSchema<ReturnType<typeof getEvalStudioCapabilitySchema>>;
+/** Form-editable Eval source field name. */
+export type EvalEditableField = InferSchema<ReturnType<typeof getEvalEditableFieldSchema>>;
+/** Source location for an Eval definition. */
+export type EvalSourceReference = InferSchema<ReturnType<typeof getEvalSourceReferenceSchema>>;
+/** Studio-editable Eval source document. */
+export type EvalSourceDocument = InferSchema<ReturnType<typeof getEvalSourceDocumentSchema>>;
+/** Eval source patch submitted by Studio forms. */
+export type EvalSourcePatch = InferSchema<ReturnType<typeof getEvalSourcePatchSchema>>;
+/** V2-ready Eval run projection. */
+export type EvalRun = InferSchema<ReturnType<typeof getEvalRunSchema>>;
+/** Options for creating a Studio source document from a discovered eval. */
+export interface CreateEvalSourceDocumentOptions {
+    sourceText?: string;
+    capabilities?: EvalStudioCapability[];
+}
+/** Create the normalized Eval document Studio can list, inspect, and edit. */
+export declare function createEvalSourceDocument(discovered: DiscoveredEval, options?: CreateEvalSourceDocumentOptions): EvalSourceDocument;
+//# sourceMappingURL=studio.d.ts.map

package/esm/src/eval/studio.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"studio.d.ts","sourceRoot":"","sources":["../../../src/src/eval/studio.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AACjE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAGrD,2CAA2C;AAC3C,eAAO,MAAM,6BAA6B,mGAEzC,CAAC;AAEF,qDAAqD;AACrD,eAAO,MAAM,0BAA0B,gKActC,CAAC;AAEF,2CAA2C;AAC3C,eAAO,MAAM,4BAA4B;;;;GAMxC,CAAC;AAEF,6DAA6D;AAC7D,eAAO,MAAM,0BAA0B;;;;;GAOtC,CAAC;AAEF,6DAA6D;AAC7D,eAAO,MAAM,0BAA0B;;;;;;;;;;;GAQtC,CAAC;AAEF,4DAA4D;AAC5D,eAAO,MAAM,yBAAyB;;;;;;;;;;;GAarC,CAAC;AAEF,yDAAyD;AACzD,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkBvC,CAAC;AAEF,+DAA+D;AAC/D,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgBpC,CAAC;AAEF,gDAAgD;AAChD,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;GAsB5B,CAAC;AAEF,oEAAoE;AACpE,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,6BAA6B,CAAC,CAAC,CAAC;AACjG,4CAA4C;AAC5C,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC;AAC3F,8CAA8C;AAC9C,MAAM,MAAM,mBAAmB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,4BAA4B,CAAC,CAAC,CAAC;AAC/F,4CAA4C;AAC5C,MAAM,MAAM,kBAAkB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,2BAA2B,CAAC,CAAC,CAAC;AAC7F,mDAAmD;AACnD,MAAM,MAAM,eAAe,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,wBAAwB,CAAC,CAAC,CAAC;AACvF,oCAAoC;AACpC,MAAM,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC,CAAC;AAEvE,4EAA4E;AAC5E,MAAM,WAAW,+BAA+B;IAC9C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,oBAAoB,EAAE,CAAC;CACvC;AAsDD,8EAA8E;AAC9E,wBAAgB,wBAAwB,CACtC,UAAU,EAAE,cAAc,EAC1B,OAAO,GAAE,+BAAoC,GAC5C,kBAAkB,CAyBpB"}

package/esm/src/eval/studio.js

@@ -0,0 +1,176 @@
+import { defineSchema } from "../schemas/index.js";
+/** Schema for Eval Studio capabilities. */
+export const getEvalStudioCapabilitySchema = defineSchema((v) => v.enum(["project.evals.read", "project.evals.write"]));
+/** Schema for an editable Eval source field name. */
+export const getEvalEditableFieldSchema = defineSchema((v) => v.enum([
+    "name",
+    "description",
+    "target",
+    "dataset",
+    "repetitions",
+    "tags",
+    "metadata",
+    "metrics",
+    "check",
+]));
+/** Schema for an Eval source reference. */
+export const getEvalSourceReferenceSchema = defineSchema((v) => v.object({
+    filePath: v.string(),
+    exportName: v.string(),
+    content: v.string().optional(),
+}));
+/** Schema for an Eval example in Studio source documents. */
+export const getEvalSourceExampleSchema = defineSchema((v) => v.object({
+    id: v.string(),
+    input: v.unknown(),
+    reference: v.unknown().optional(),
+    metadata: v.record(v.string(), v.unknown()).optional(),
+}));
+/** Schema for an Eval dataset in Studio source documents. */
+export const getEvalSourceDatasetSchema = defineSchema((v) => v.object({
+    kind: v.enum(["inline", "json", "jsonl", "dynamic"]),
+    path: v.string().optional(),
+    examples: v.array(getEvalSourceExampleSchema()).optional(),
+    editable: v.boolean(),
+    dynamic: v.boolean(),
+}));
+/** Schema for an Eval metric in Studio source documents. */
+export const getEvalSourceMetricSchema = defineSchema((v) => v.object({
+    name: v.string(),
+    family: v.enum(["answer", "agent", "ops", "judge", "check"]),
+    severity: v.enum(["gate", "soft", "budget"]),
+    threshold: v.object({
+        min: v.number().optional(),
+        max: v.number().optional(),
+    }).optional(),
+    config: v.record(v.string(), v.unknown()).optional(),
+    editable: v.boolean(),
+    dynamic: v.boolean(),
+}));
+/** Schema for a Studio-editable Eval source document. */
+export const getEvalSourceDocumentSchema = defineSchema((v) => v.object({
+    kind: v.literal("eval-source-document"),
+    id: v.string(),
+    name: v.string(),
+    description: v.string().optional(),
+    targetKind: v.literal("agent"),
+    target: v.string(),
+    source: getEvalSourceReferenceSchema(),
+    dataset: getEvalSourceDatasetSchema(),
+    metrics: v.array(getEvalSourceMetricSchema()),
+    repetitions: v.number().int().positive(),
+    tags: v.array(v.string()),
+    metadata: v.record(v.string(), v.unknown()),
+    editableFields: v.array(getEvalEditableFieldSchema()),
+    dynamicFields: v.array(getEvalEditableFieldSchema()),
+    capabilities: v.array(getEvalStudioCapabilitySchema()),
+}));
+/** Schema for a source patch submitted from an Eval editor. */
+export const getEvalSourcePatchSchema = defineSchema((v) => v.object({
+    kind: v.literal("eval-source-patch"),
+    id: v.string(),
+    source: getEvalSourceReferenceSchema(),
+    fields: v.object({
+        name: v.string().optional(),
+        description: v.string().optional(),
+        target: v.string().optional(),
+        dataset: getEvalSourceDatasetSchema().optional(),
+        repetitions: v.number().int().positive().optional(),
+        tags: v.array(v.string()).optional(),
+        metadata: v.record(v.string(), v.unknown()).optional(),
+        metrics: v.array(getEvalSourceMetricSchema()).optional(),
+    }).strict(),
+}));
+/** Schema for V2-ready Eval run projections. */
+export const getEvalRunSchema = defineSchema((v) => v.object({
+    kind: v.literal("eval-run"),
+    runId: v.string(),
+    evalId: v.string(),
+    status: v.enum(["pending", "running", "waiting", "completed", "failed", "cancelled"]),
+    targetKind: v.literal("agent"),
+    target: v.string(),
+    source: getEvalSourceReferenceSchema().optional(),
+    summary: v.object({
+        records: v.number().int().nonnegative(),
+        passed: v.number().int().nonnegative(),
+        failed: v.number().int().nonnegative(),
+        passRate: v.number(),
+    }).nullable(),
+    reportPath: v.string().nullable(),
+    error: v.unknown().nullable(),
+    metadata: v.record(v.string(), v.unknown()),
+    createdAt: v.string(),
+    startedAt: v.string().nullable(),
+    completedAt: v.string().nullable(),
+}));
+const DEFAULT_EVAL_STUDIO_CAPABILITIES = [
+    "project.evals.read",
+    "project.evals.write",
+];
+const BASE_EDITABLE_FIELDS = [
+    "name",
+    "description",
+    "target",
+    "dataset",
+    "repetitions",
+    "tags",
+    "metadata",
+    "metrics",
+];
+function normalizeDataset(definition) {
+    const dataset = definition.dataset;
+    const dynamic = !["inline", "json", "jsonl"].includes(dataset.kind);
+    return {
+        kind: dynamic ? "dynamic" : dataset.kind,
+        ...(dataset.path ? { path: dataset.path } : {}),
+        ...(dataset.examples ? { examples: dataset.examples.map(copyExample) } : {}),
+        editable: !dynamic,
+        dynamic,
+    };
+}
+function copyExample(example) {
+    return {
+        id: example.id,
+        input: example.input,
+        ...(example.reference !== undefined ? { reference: example.reference } : {}),
+        ...(example.metadata ? { metadata: { ...example.metadata } } : {}),
+    };
+}
+function createSourceMetric(metric) {
+    const dynamic = metric.family === "judge" || metric.family === "check";
+    return {
+        name: metric.name,
+        family: metric.family,
+        severity: metric.severity,
+        ...(metric.threshold ? { threshold: { ...metric.threshold } } : {}),
+        ...(metric.config ? { config: { ...metric.config } } : {}),
+        editable: !!metric.config || metric.name === "agent.noFailedTools",
+        dynamic,
+    };
+}
+/** Create the normalized Eval document Studio can list, inspect, and edit. */
+export function createEvalSourceDocument(discovered, options = {}) {
+    const { definition } = discovered;
+    const dynamicFields = definition.check ? ["check"] : [];
+    return getEvalSourceDocumentSchema().parse({
+        kind: "eval-source-document",
+        id: discovered.id,
+        name: definition.name,
+        ...(definition.description ? { description: definition.description } : {}),
+        targetKind: definition.targetKind,
+        target: definition.target,
+        source: {
+            filePath: discovered.filePath,
+            exportName: discovered.exportName,
+            ...(options.sourceText ? { content: options.sourceText } : {}),
+        },
+        dataset: normalizeDataset(definition),
+        metrics: definition.metrics.map(createSourceMetric),
+        repetitions: definition.repetitions,
+        tags: [...definition.tags],
+        metadata: { ...definition.metadata },
+        editableFields: BASE_EDITABLE_FIELDS,
+        dynamicFields,
+        capabilities: options.capabilities ?? DEFAULT_EVAL_STUDIO_CAPABILITIES,
+    });
+}

package/esm/src/eval/types.d.ts

@@ -0,0 +1,221 @@
+/**
+ * Types for Veryfront eval definitions, datasets, metrics, and reports.
+ *
+ * @module eval
+ */
+/** Primitive kind an eval can execute. V1 supports agent targets. */
+export type EvalTargetKind = "agent";
+/** How a metric result affects the final eval result. */
+export type EvalSeverity = "gate" | "soft" | "budget";
+/** Metric family used for grouping report summaries. */
+export type EvalMetricFamily = "answer" | "agent" | "ops" | "judge" | "check";
+/** Numeric threshold attached to score-based metrics. */
+export type EvalMetricThreshold = {
+    min?: number;
+    max?: number;
+};
+/** Value that can be returned synchronously or as a promise. */
+export type EvalMaybePromise<T> = T | Promise<T>;
+/** Normalized dataset example used by eval runners and reports. */
+export interface EvalExample {
+    id: string;
+    input: unknown;
+    reference?: unknown;
+    metadata?: Record<string, unknown>;
+}
+/** Dataset example shape accepted by eval definitions. */
+export type EvalExampleInput = {
+    id: string;
+    input: unknown;
+    reference?: unknown;
+    metadata?: Record<string, unknown>;
+};
+/** Context passed to dataset loaders. */
+export interface EvalDatasetLoadContext {
+    baseDir: string;
+}
+/** Dataset loader used by an eval definition. */
+export interface EvalDataset {
+    kind: "inline" | "json" | "jsonl";
+    path?: string;
+    examples?: EvalExample[];
+    load(context: EvalDatasetLoadContext): Promise<EvalExample[]>;
+}
+/** Token and cost usage captured for one eval record. */
+export interface EvalUsage {
+    inputTokens?: number;
+    outputTokens?: number;
+    totalTokens?: number;
+    costUsd?: number;
+}
+/** Tool call metadata captured during one eval record. */
+export interface EvalToolCall {
+    name: string;
+    status?: "ok" | "error" | "skipped";
+    error?: string;
+    metadata?: Record<string, unknown>;
+}
+/** Trace metadata captured for one eval record. */
+export interface EvalTrace {
+    events: unknown[];
+    toolCalls: EvalToolCall[];
+}
+/** One executed example and repetition inside an eval report. */
+export interface EvalRecord {
+    id: string;
+    evalId: string;
+    exampleId: string;
+    repetition: number;
+    input: unknown;
+    output: unknown;
+    reference?: unknown;
+    metadata: Record<string, unknown>;
+    trace: EvalTrace;
+    usage: EvalUsage;
+    durationMs: number;
+    completed: boolean;
+    error?: string;
+    metrics?: EvalMetricResult[];
+    checks?: EvalMetricResult[];
+}
+/** Result emitted by a metric or check assertion. */
+export interface EvalMetricResult {
+    name: string;
+    family: EvalMetricFamily;
+    severity: EvalSeverity;
+    score?: number;
+    pass?: boolean;
+    skipped?: boolean;
+    label?: string;
+    explanation?: string;
+    evidence?: Record<string, unknown>;
+}
+/** Optional runtime context passed to metric evaluators. */
+export interface EvalMetricContext {
+    now?: () => Date;
+}
+/** Metric contract used by eval definitions. */
+export interface EvalMetric {
+    name: string;
+    family: EvalMetricFamily;
+    severity: EvalSeverity;
+    threshold?: EvalMetricThreshold;
+    config?: Record<string, unknown>;
+    evaluate(record: EvalRecord, context?: EvalMetricContext): EvalMaybePromise<EvalMetricResult>;
+    gate(threshold?: EvalMetricThreshold): EvalMetric;
+    soft(threshold?: EvalMetricThreshold): EvalMetric;
+    budget(threshold?: EvalMetricThreshold): EvalMetric;
+}
+/** Fluent severity helpers for `check` expectations. */
+export interface EvalExpectation {
+    gate(threshold?: EvalMetricThreshold): EvalMetricResult;
+    soft(threshold?: EvalMetricThreshold): EvalMetricResult;
+    budget(threshold?: EvalMetricThreshold): EvalMetricResult;
+}
+/** Built-in expectation helpers available inside `check`. */
+export interface EvalExpect {
+    completed(): EvalExpectation;
+    outputContains(text: string): EvalExpectation;
+    noFailedTools(): EvalExpectation;
+}
+/** Context passed to an eval definition's `check` callback. */
+export interface EvalCheckContext {
+    definition: EvalDefinition;
+    example: EvalExample;
+    repetition: number;
+    record: EvalRecord;
+    expect: EvalExpect;
+}
+/** Source location for a discovered eval definition. */
+export interface EvalSource {
+    filePath: string;
+    exportName: string;
+}
+/** First-class eval definition discovered from project source. */
+export interface EvalDefinition {
+    kind: "eval";
+    targetKind: EvalTargetKind;
+    id: string;
+    name: string;
+    description?: string;
+    target: string;
+    dataset: EvalDataset;
+    metrics: EvalMetric[];
+    repetitions: number;
+    tags: string[];
+    metadata: Record<string, unknown>;
+    source?: EvalSource;
+    check?: (context: EvalCheckContext) => EvalMaybePromise<void>;
+}
+/** Input accepted by `evalAgent`. */
+export interface EvalAgentInput {
+    id?: string;
+    name?: string;
+    description?: string;
+    target: string;
+    dataset: EvalDataset | EvalExampleInput[];
+    metrics?: EvalMetric[];
+    repetitions?: number;
+    tags?: string[];
+    metadata?: Record<string, unknown>;
+    check?: (context: EvalCheckContext) => EvalMaybePromise<void>;
+}
+/** Context passed to an agent adapter when `runEval` executes an example. */
+export interface EvalAgentAdapterContext {
+    definition: EvalDefinition;
+    example: EvalExample;
+    repetition: number;
+}
+/** Agent adapter result normalized into an eval record. */
+export interface EvalAgentAdapterResult {
+    text?: string;
+    json?: unknown;
+    output?: unknown;
+    trace?: Partial<EvalTrace>;
+    usage?: EvalUsage;
+    durationMs?: number;
+    completed?: boolean;
+    error?: string;
+}
+/** Adapter used by `runEval` to execute V1 agent targets. */
+export type EvalAgentAdapter = (context: EvalAgentAdapterContext) => EvalMaybePromise<string | EvalAgentAdapterResult>;
+/** Options for running an eval locally. */
+export interface RunEvalOptions {
+    adapters: {
+        agent: EvalAgentAdapter;
+    };
+    baseDir?: string;
+    runId?: string;
+    now?: () => Date;
+}
+/** Aggregate pass/fail summary for one metric. */
+export interface EvalMetricSummary {
+    name: string;
+    family: EvalMetricFamily;
+    severity: EvalSeverity;
+    passed: number;
+    failed: number;
+    skipped: number;
+    passRate: number;
+}
+/** Aggregate pass/fail summary for one eval report. */
+export interface EvalReportSummary {
+    records: number;
+    passed: number;
+    failed: number;
+    passRate: number;
+    metrics: EvalMetricSummary[];
+}
+/** JSON-serializable report produced by `runEval`. */
+export interface EvalReport {
+    kind: "eval-report";
+    runId: string;
+    definitionId: string;
+    targetKind: EvalTargetKind;
+    target: string;
+    startedAt: string;
+    endedAt: string;
+    summary: EvalReportSummary;
+    records: EvalRecord[];
+}
+//# sourceMappingURL=types.d.ts.map

package/esm/src/eval/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,qEAAqE;AACrE,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC;AAErC,yDAAyD;AACzD,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEtD,wDAAwD;AACxD,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,OAAO,GAAG,KAAK,GAAG,OAAO,GAAG,OAAO,CAAC;AAE9E,yDAAyD;AACzD,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,gEAAgE;AAChE,MAAM,MAAM,gBAAgB,CAAC,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;AAEjD,mEAAmE;AACnE,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,0DAA0D;AAC1D,MAAM,MAAM,gBAAgB,GAAG;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,CAAC;AAEF,yCAAyC;AACzC,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,iDAAiD;AACjD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC;IAClC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,WAAW,EAAE,CAAC;IACzB,IAAI,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;CAC/D;AAED,yDAAyD;AACzD,MAAM,WAAW,SAAS;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,0DAA0D;AAC1D,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,IAAI,GAAG,OAAO,GAAG,SAAS,CAAC;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,mDAAmD;AACnD,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,OAAO,EAAE,CAAC;IAClB,SAAS,EAAE,YAAY,EAAE,CAAC;CAC3B;AAED,iEAAiE;AACjE,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC7B,MAAM,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC7B;AAED,qDAAqD;AACrD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,4DAA4D;AAC5D,MAAM,WAAW,iBAAiB;IAChC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB;AAED,gDAAgD;AAChD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,SAAS,CAAC,EAAE,mBAAmB,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,CAAC;IAC9F,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,UAAU,CAAC;IAClD,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,UAAU,CAAC;IAClD,MAAM,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,UAAU,CAAC;CACrD;AAED,wDAAwD;AACxD,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,gBAAgB,CAAC;IACxD,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,gBAAgB,CAAC;IACxD,MAAM,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,gBAAgB,CAAC;CAC3D;AAED,6DAA6D;AAC7D,MAAM,WAAW,UAAU;IACzB,SAAS,IAAI,eAAe,CAAC;IAC7B,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,CAAC;IAC9C,aAAa,IAAI,eAAe,CAAC;CAClC;AAED,+DAA+D;AAC/D,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;CACpB;AAED,wDAAwD;AACxD,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,kEAAkE;AAClE,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,WAAW,CAAC;IACrB,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,KAAK,gBAAgB,CAAC,IAAI,CAAC,CAAC;CAC/D;AAED,qCAAqC;AACrC,MAAM,WAAW,cAAc;IAC7B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,WAAW,GAAG,gBAAgB,EAAE,CAAC;IAC1C,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,KAAK,gBAAgB,CAAC,IAAI,CAAC,CAAC;CAC/D;AAED,6EAA6E;AAC7E,MAAM,WAAW,uBAAuB;IACtC,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,2DAA2D;AAC3D,MAAM,WAAW,sBAAsB;IACrC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC3B,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,6DAA6D;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAC7B,OAAO,EAAE,uBAAuB,KAC7B,gBAAgB,CAAC,MAAM,GAAG,sBAAsB,CAAC,CAAC;AAEvD,2CAA2C;AAC3C,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE;QACR,KAAK,EAAE,gBAAgB,CAAC;KACzB,CAAC;IACF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB;AAED,kDAAkD;AAClD,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,uDAAuD;AACvD,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,iBAAiB,EAAE,CAAC;CAC9B;AAED,sDAAsD;AACtD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,cAAc,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB"}

package/esm/src/eval/types.js

@@ -0,0 +1,6 @@
+/**
+ * Types for Veryfront eval definitions, datasets, metrics, and reports.
+ *
+ * @module eval
+ */
+export {};

package/esm/src/eval/validation.d.ts

@@ -0,0 +1,4 @@
+import type { EvalExample, EvalExampleInput } from "./types.js";
+export declare function createEvalValidationError(message: string): Error;
+export declare function normalizeEvalExamples(examples: readonly EvalExampleInput[], source: string): EvalExample[];
+//# sourceMappingURL=validation.d.ts.map

package/esm/src/eval/validation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validation.d.ts","sourceRoot":"","sources":["../../../src/src/eval/validation.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEhE,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,CAEhE;AAWD,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,SAAS,gBAAgB,EAAE,EACrC,MAAM,EAAE,MAAM,GACb,WAAW,EAAE,CAmCf"}

package/esm/src/eval/validation.js

@@ -0,0 +1,41 @@
+import { INVALID_ARGUMENT } from "../errors/index.js";
+export function createEvalValidationError(message) {
+    return INVALID_ARGUMENT.create({ message });
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function assertMetadata(value, label) {
+    if (value === undefined || isRecord(value))
+        return;
+    throw createEvalValidationError(`${label} metadata must be an object when provided`);
+}
+export function normalizeEvalExamples(examples, source) {
+    if (!Array.isArray(examples)) {
+        throw createEvalValidationError(`${source} must be an array of eval examples`);
+    }
+    const seenIds = new Set();
+    return examples.map((example, index) => {
+        if (!isRecord(example)) {
+            throw createEvalValidationError(`${source}[${index}] must be an object`);
+        }
+        const id = example.id;
+        if (typeof id !== "string" || id.trim() === "") {
+            throw createEvalValidationError(`${source}[${index}] id must be a non-empty string`);
+        }
+        if (seenIds.has(id)) {
+            throw createEvalValidationError(`Duplicate eval example id "${id}" in ${source}`);
+        }
+        seenIds.add(id);
+        if (!Object.hasOwn(example, "input")) {
+            throw createEvalValidationError(`${source}[${index}] input is required`);
+        }
+        assertMetadata(example.metadata, `${source}[${index}]`);
+        return {
+            id,
+            input: example.input,
+            ...(Object.hasOwn(example, "reference") ? { reference: example.reference } : {}),
+            ...(example.metadata ? { metadata: example.metadata } : {}),
+        };
+    });
+}