vieval 0.0.11 → 0.0.12

package/dist/config.d.mts

@@ -1,2 +1,2 @@
-import { C as TaskDefinition, D as TaskRunContext, E as TaskReporterHooks, O as TaskRunOutput, R as ModelDefinition, S as TaskConcurrencyConfig, T as TaskReporterEventPayload, _ as ScopedMatrices, a as CliOpenTelemetryReportingConfig, b as TaskCaseReporterPayload, c as EvalDefinition, d as MatrixAxisValues, f as MatrixDefinition, g as MatrixValue, h as MatrixRow, i as Awaitable, l as EvalModule, m as MatrixPrimitive, n as defineEval, o as CliReportingConfig, p as MatrixLayer, r as defineTask, s as CollectedEvalEntry, t as ConfigHookPlugin, u as EvalModuleMap, v as TaskAutoRetryDelay, w as TaskExecutionPolicy, x as TaskCaseState, y as TaskCaseReporterEndPayload, z as resolveModelByName } from "./index-CwKBlCG9.mjs";
+import { C as TaskDefinition, D as TaskRunContext, E as TaskReporterHooks, O as TaskRunOutput, R as ModelDefinition, S as TaskConcurrencyConfig, T as TaskReporterEventPayload, _ as ScopedMatrices, a as CliOpenTelemetryReportingConfig, b as TaskCaseReporterPayload, c as EvalDefinition, d as MatrixAxisValues, f as MatrixDefinition, g as MatrixValue, h as MatrixRow, i as Awaitable, l as EvalModule, m as MatrixPrimitive, n as defineEval, o as CliReportingConfig, p as MatrixLayer, r as defineTask, s as CollectedEvalEntry, t as ConfigHookPlugin, u as EvalModuleMap, v as TaskAutoRetryDelay, w as TaskExecutionPolicy, x as TaskCaseState, y as TaskCaseReporterEndPayload, z as resolveModelByName } from "./index-BLIlhiWT.mjs";
 export { Awaitable, CliOpenTelemetryReportingConfig, CliReportingConfig, CollectedEvalEntry, ConfigHookPlugin, EvalDefinition, EvalModule, EvalModuleMap, MatrixAxisValues, MatrixDefinition, MatrixLayer, MatrixPrimitive, MatrixRow, MatrixValue, ModelDefinition, ScopedMatrices, TaskAutoRetryDelay, TaskCaseReporterEndPayload, TaskCaseReporterPayload, TaskCaseState, TaskConcurrencyConfig, TaskDefinition, TaskExecutionPolicy, TaskReporterEventPayload, TaskReporterHooks, TaskRunContext, TaskRunOutput, defineEval, defineTask, resolveModelByName };

package/dist/core/assertions/index.d.mts

@@ -1,47 +1,30 @@
-import { X as RunScoreKind, Y as RunScore } from "../../index-CwKBlCG9.mjs";
+import { X as RunScoreKind, Y as RunScore } from "../../index-BLIlhiWT.mjs";
 
 //#region src/core/assertions/index.d.ts
 /**
- * Stores mutable evaluation state for stateful assertion flows.
- *
- * Use when:
- * - assertions need to share counters, rolling metrics, or memoized values
- * - a scenario evaluates multiple steps and expects state-aware checks
- */
-type AssertionState = Map<string, unknown>;
-/**
- * Represents one tool call emitted by a model response.
+ * Async assertion function used by eval scenarios.
  */
-interface ToolCall {
-  /**
-   * Tool name used by the call.
-   */
-  name: string;
-  /**
-   * Tool arguments payload.
-   */
-  args: unknown;
-}
+type Assertion = (context: AssertionContext) => Promise<AssertionOutcome>;
 /**
  * Normalized assertion context for one model output.
  */
 interface AssertionContext {
   /**
-   * Plain text model output used by text assertions.
+   * Shared mutable state for stateful assertion measurement.
    */
-  text: string;
+  state: AssertionState;
   /**
    * Optional structured output parsed from the model response.
    */
   structuredOutput?: unknown;
   /**
-   * Optional tool calls extracted from the model response.
+   * Plain text model output used by text assertions.
    */
-  toolCalls?: readonly ToolCall[];
+  text: string;
   /**
-   * Shared mutable state for stateful assertion measurement.
+   * Optional tool calls extracted from the model response.
    */
-  state: AssertionState;
+  toolCalls?: readonly ToolCall[];
 }
 /**
  * Result for one assertion evaluation.
@@ -51,70 +34,66 @@ interface AssertionOutcome {
    * Stable assertion id.
    */
   id: string;
-  /**
-   * Assertion family emitted as run score kind.
-   */
-  scoreKind: RunScoreKind;
   /**
    * Whether the assertion passed.
    */
   pass: boolean;
+  /**
+   * Human-readable reason for logs and reports.
+   */
+  reason: string;
   /**
    * Normalized score in the `0..1` range.
    */
   score: number;
   /**
-   * Human-readable reason for logs and reports.
+   * Assertion family emitted as run score kind.
    */
-  reason: string;
+  scoreKind: RunScoreKind;
 }
 /**
- * Async assertion function used by eval scenarios.
- */
-type Assertion = (context: AssertionContext) => Promise<AssertionOutcome>;
-/**
- * Normalizes text for matching.
+ * Stores mutable evaluation state for stateful assertion flows.
  *
- * Before: `"  Hello\nWorld  "`
- * After: `"hello world"`
+ * Use when:
+ * - assertions need to share counters, rolling metrics, or memoized values
+ * - a scenario evaluates multiple steps and expects state-aware checks
  */
-declare function normalizeMatchText(value: string, caseSensitive: boolean): string;
+type AssertionState = Map<string, unknown>;
 /**
- * Options for include-keyword assertions.
+ * Options for custom assertions.
  */
-interface MustIncludeAssertionOptions {
+interface CustomAssertionOptions {
   /**
-   * Stable assertion id.
+   * Custom evaluator callback.
    */
-  id: string;
+  evaluate: (context: AssertionContext) => Promise<{
+    pass: boolean;
+    reason: string;
+    score: number;
+  }> | {
+    pass: boolean;
+    reason: string;
+    score: number;
+  };
   /**
-   * Keywords that must be present.
+   * Stable assertion id.
    */
-  keywords: readonly string[];
+  id: string;
   /**
-   * Match mode for keywords.
-   *
-   * @default 'all'
+   * Score family emitted by this custom assertion.
    */
-  mode?: 'all' | 'any';
+  scoreKind: RunScoreKind;
+}
+/**
+ * Options for exclude-keyword assertions.
+ */
+interface MustExcludeAssertionOptions {
   /**
    * Case-sensitive matching toggle.
    *
    * @default false
    */
   caseSensitive?: boolean;
-}
-/**
- * Creates an assertion that requires specific keywords in model text.
- *
- * Example:
- * `expectMustInclude({ id: 'tone', keywords: ['calm', 'move'] })`
- */
-declare function expectMustInclude(options: MustIncludeAssertionOptions): Assertion;
-/**
- * Options for exclude-keyword assertions.
- */
-interface MustExcludeAssertionOptions {
   /**
    * Stable assertion id.
    */
@@ -123,156 +102,144 @@ interface MustExcludeAssertionOptions {
    * Keywords that must not appear.
    */
   keywords: readonly string[];
+}
+/**
+ * Options for include-keyword assertions.
+ */
+interface MustIncludeAssertionOptions {
   /**
    * Case-sensitive matching toggle.
    *
    * @default false
    */
   caseSensitive?: boolean;
-}
-/**
- * Creates an assertion that forbids specific keywords.
- *
- * Example:
- * `expectMustExclude({ id: 'no-engine-dump', keywords: ['bestmove', 'ponder'] })`
- */
-declare function expectMustExclude(options: MustExcludeAssertionOptions): Assertion;
-/**
- * Options for regular-expression assertions.
- */
-interface RegexAssertionOptions {
   /**
    * Stable assertion id.
    */
   id: string;
   /**
-   * Pattern to apply to model text.
+   * Keywords that must be present.
    */
-  pattern: RegExp;
+  keywords: readonly string[];
+  /**
+   * Match mode for keywords.
+   *
+   * @default 'all'
+   */
+  mode?: 'all' | 'any';
 }
 /**
- * Creates an assertion based on a regular expression.
- *
- * Example:
- * `expectRegex({ id: 'starts-with-act', pattern: /^<\|ACT:/ })`
- */
-declare function expectRegex(options: RegexAssertionOptions): Assertion;
-/**
- * Options for structured-output assertions.
+ * Options for regular-expression assertions.
  */
-interface StructuredOutputAssertionOptions<TValue> {
+interface RegexAssertionOptions {
   /**
    * Stable assertion id.
    */
   id: string;
   /**
-   * Runtime validator for structured output.
-   */
-  validate: (value: unknown) => value is TValue;
-  /**
-   * Optional failure reason.
+   * Pattern to apply to model text.
    */
-  failureReason?: string;
+  pattern: RegExp;
 }
 /**
- * Creates an assertion for structured model output.
- *
- * Example:
- * `expectStructuredOutput({ id: 'json-shape', validate: isMySchema })`
- */
-declare function expectStructuredOutput<TValue>(options: StructuredOutputAssertionOptions<TValue>): Assertion;
-/**
- * Options for tool-call argument assertions.
+ * Options for rubric assertions.
  */
-interface ToolCallArgsAssertionOptions {
+interface RubricAssertionOptions {
   /**
    * Stable assertion id.
    */
   id: string;
   /**
-   * Tool name to inspect.
+   * Async rubric judge callback.
    */
-  toolName: string;
+  judge: (context: AssertionContext) => Promise<RubricJudgeResult>;
   /**
-   * Runtime validator for tool arguments.
+   * Minimum passing score.
+   *
+   * @default 0.7
    */
-  validate: (args: unknown) => boolean;
+  minScore?: number;
 }
-/**
- * Creates an assertion for validating tool-call arguments.
- *
- * Example:
- * `expectToolCallArgs({ id: 'spark-command-shape', toolName: 'builtIn_sparkCommand', validate: isSparkArgs })`
- */
-declare function expectToolCallArgs(options: ToolCallArgsAssertionOptions): Assertion;
 /**
  * Rubric judge result returned by teacher-model or rubric logic.
  */
 interface RubricJudgeResult {
   /**
-   * Normalized score in the `0..1` range.
+   * Optional judge model id.
    */
-  score: number;
+  judgeModel?: string;
   /**
    * Judge explanation text.
    */
   reason: string;
   /**
-   * Optional judge model id.
+   * Normalized score in the `0..1` range.
    */
-  judgeModel?: string;
+  score: number;
 }
 /**
- * Options for rubric assertions.
+ * Options for structured-output assertions.
  */
-interface RubricAssertionOptions {
+interface StructuredOutputAssertionOptions<TValue> {
   /**
-   * Stable assertion id.
+   * Optional failure reason.
    */
-  id: string;
+  failureReason?: string;
   /**
-   * Async rubric judge callback.
+   * Stable assertion id.
    */
-  judge: (context: AssertionContext) => Promise<RubricJudgeResult>;
+  id: string;
   /**
-   * Minimum passing score.
-   *
-   * @default 0.7
+   * Runtime validator for structured output.
    */
-  minScore?: number;
+  validate: (value: unknown) => value is TValue;
 }
 /**
- * Creates a rubric assertion driven by teacher-model style scoring.
- *
- * Example:
- * `expectRubric({ id: 'human-like-tone', judge: judgeFn, minScore: 0.8 })`
+ * Represents one tool call emitted by a model response.
  */
-declare function expectRubric(options: RubricAssertionOptions): Assertion;
+interface ToolCall {
+  /**
+   * Tool arguments payload.
+   */
+  args: unknown;
+  /**
+   * Tool name used by the call.
+   */
+  name: string;
+}
 /**
- * Options for custom assertions.
+ * Options for tool-call argument assertions.
  */
-interface CustomAssertionOptions {
+interface ToolCallArgsAssertionOptions {
   /**
    * Stable assertion id.
    */
   id: string;
   /**
-   * Score family emitted by this custom assertion.
+   * Tool name to inspect.
    */
-  scoreKind: RunScoreKind;
+  toolName: string;
   /**
-   * Custom evaluator callback.
+   * Runtime validator for tool arguments.
    */
-  evaluate: (context: AssertionContext) => Promise<{
-    pass: boolean;
-    reason: string;
-    score: number;
-  }> | {
-    pass: boolean;
-    reason: string;
-    score: number;
-  };
+  validate: (args: unknown) => boolean;
 }
+/**
+ * Returns failing assertion outcomes in original order.
+ */
+declare function collectFailedAssertions(outcomes: readonly AssertionOutcome[]): AssertionOutcome[];
+/**
+ * Executes assertion list and returns all outcomes.
+ *
+ * Call stack:
+ *
+ * {@link evaluateAssertions}
+ *   -> `assertion(context)`
+ *     -> {@link AssertionOutcome}[]
+ */
+declare function evaluateAssertions(assertions: readonly Assertion[], context: Omit<AssertionContext, 'state'> & {
+  state?: AssertionState;
+}): Promise<AssertionOutcome[]>;
 /**
  * Creates a custom assertion with fully user-defined logic.
  *
@@ -280,6 +247,20 @@ interface CustomAssertionOptions {
  * `expectCustom({ id: 'stateful-window', scoreKind: 'exact', evaluate: (ctx) => ... })`
  */
 declare function expectCustom(options: CustomAssertionOptions): Assertion;
+/**
+ * Creates an assertion that forbids specific keywords.
+ *
+ * Example:
+ * `expectMustExclude({ id: 'no-engine-dump', keywords: ['bestmove', 'ponder'] })`
+ */
+declare function expectMustExclude(options: MustExcludeAssertionOptions): Assertion;
+/**
+ * Creates an assertion that requires specific keywords in model text.
+ *
+ * Example:
+ * `expectMustInclude({ id: 'tone', keywords: ['calm', 'move'] })`
+ */
+declare function expectMustInclude(options: MustIncludeAssertionOptions): Assertion;
 /**
  * Creates an inverse assertion.
  *
@@ -290,25 +271,44 @@ declare function expectNot(assertion: Assertion, options: {
   id: string;
 }): Assertion;
 /**
- * Executes assertion list and returns all outcomes.
+ * Creates an assertion based on a regular expression.
  *
- * Call stack:
+ * Example:
+ * `expectRegex({ id: 'starts-with-act', pattern: /^<\|ACT:/ })`
+ */
+declare function expectRegex(options: RegexAssertionOptions): Assertion;
+/**
+ * Creates a rubric assertion driven by teacher-model style scoring.
  *
- * {@link evaluateAssertions}
- *   -> `assertion(context)`
- *     -> {@link AssertionOutcome}[]
+ * Example:
+ * `expectRubric({ id: 'human-like-tone', judge: judgeFn, minScore: 0.8 })`
  */
-declare function evaluateAssertions(assertions: readonly Assertion[], context: Omit<AssertionContext, 'state'> & {
-  state?: AssertionState;
-}): Promise<AssertionOutcome[]>;
+declare function expectRubric(options: RubricAssertionOptions): Assertion;
 /**
- * Converts assertion outcomes to run-score tuples consumed by aggregation.
+ * Creates an assertion for structured model output.
+ *
+ * Example:
+ * `expectStructuredOutput({ id: 'json-shape', validate: isMySchema })`
  */
-declare function toRunScores(outcomes: readonly AssertionOutcome[]): RunScore[];
+declare function expectStructuredOutput<TValue>(options: StructuredOutputAssertionOptions<TValue>): Assertion;
 /**
- * Returns failing assertion outcomes in original order.
+ * Creates an assertion for validating tool-call arguments.
+ *
+ * Example:
+ * `expectToolCallArgs({ id: 'spark-command-shape', toolName: 'builtIn_sparkCommand', validate: isSparkArgs })`
  */
-declare function collectFailedAssertions(outcomes: readonly AssertionOutcome[]): AssertionOutcome[];
+declare function expectToolCallArgs(options: ToolCallArgsAssertionOptions): Assertion;
+/**
+ * Normalizes text for matching.
+ *
+ * Before: `"  Hello\nWorld  "`
+ * After: `"hello world"`
+ */
+declare function normalizeMatchText(value: string, caseSensitive: boolean): string;
+/**
+ * Converts assertion outcomes to run-score tuples consumed by aggregation.
+ */
+declare function toRunScores(outcomes: readonly AssertionOutcome[]): RunScore[];
 //#endregion
 export { Assertion, AssertionContext, AssertionOutcome, AssertionState, CustomAssertionOptions, MustExcludeAssertionOptions, MustIncludeAssertionOptions, RegexAssertionOptions, RubricAssertionOptions, RubricJudgeResult, StructuredOutputAssertionOptions, ToolCall, ToolCallArgsAssertionOptions, collectFailedAssertions, evaluateAssertions, expectCustom, expectMustExclude, expectMustInclude, expectNot, expectRegex, expectRubric, expectStructuredOutput, expectToolCallArgs, normalizeMatchText, toRunScores };
 //# sourceMappingURL=index.d.mts.map