vieval 0.0.1 → 0.0.3

package/dist/plugins/chat-models/index.d.mts

@@ -1,5 +1,288 @@
-import { n as ModelDefinition, t as ConfigHookPlugin } from "../../plugin-DVaRZY2x.mjs";
+import { M as ModelDefinition, l as MatrixDefinition, t as ConfigHookPlugin, x as TaskRunContext } from "../../index-BDMEAmf2.mjs";
 
+//#region src/plugins/chat-models/runtime-config.d.ts
+/**
+ * Runtime config consumed by OpenAI-compatible provider constructors.
+ */
+interface OpenAIChatModelRuntimeConfig {
+  /**
+   * Resolved inference executor kind.
+   */
+  inferenceExecutor: 'openai';
+  /**
+   * Concrete model name.
+   */
+  model: string;
+  /**
+   * Required API key.
+   */
+  apiKey: string;
+  /**
+   * Optional base URL override.
+   */
+  baseURL?: string;
+  /**
+   * Optional request headers.
+   */
+  headers?: ChatModelHeaders;
+}
+/**
+ * Runtime config consumed by Ollama provider constructors.
+ */
+interface OllamaChatModelRuntimeConfig {
+  /**
+   * Resolved inference executor kind.
+   */
+  inferenceExecutor: 'ollama';
+  /**
+   * Concrete model name.
+   */
+  model: string;
+  /**
+   * Optional base URL override.
+   */
+  baseURL?: string;
+  /**
+   * Optional request headers.
+   */
+  headers?: ChatModelHeaders;
+}
+/**
+ * Runtime config consumed by OpenRouter provider constructors.
+ */
+interface OpenRouterChatModelRuntimeConfig {
+  /**
+   * Resolved inference executor kind.
+   */
+  inferenceExecutor: 'openrouter';
+  /**
+   * Concrete model name.
+   */
+  model: string;
+  /**
+   * Required API key.
+   */
+  apiKey: string;
+  /**
+   * Optional base URL override.
+   */
+  baseURL?: string;
+  /**
+   * Optional request headers.
+   */
+  headers?: ChatModelHeaders;
+}
+/**
+ * Union of normalized runtime configs for supported chat-model executors.
+ */
+type ChatModelRuntimeConfig = OpenAIChatModelRuntimeConfig | OllamaChatModelRuntimeConfig | OpenRouterChatModelRuntimeConfig;
+/**
+ * Normalizes one configured chat model into runtime executor config.
+ *
+ * Use when:
+ * - eval code needs typed provider constructor options from `context.model()`
+ * - model parameters should be validated once with clear error messages
+ *
+ * Expects:
+ * - `model.inferenceExecutorId` to be one of the supported executor ids
+ * - required OpenAI fields (apiKey) to exist in `model.parameters`
+ *
+ * Returns:
+ * - validated runtime config union for OpenAI or Ollama
+ */
+declare function toChatModelRuntimeConfig(model: ModelDefinition): ChatModelRuntimeConfig;
+/**
+ * Resolves OpenAI runtime config from one resolved run-context model.
+ *
+ * Use when:
+ * - task execution already has `context.model()` output
+ * - eval code wants typed OpenAI provider options with a concise helper name
+ *
+ * Expects:
+ * - `model` to resolve to an OpenAI-backed chat model
+ *
+ * Returns:
+ * - validated OpenAI runtime config
+ */
+declare function openaiFromRunContext(model: ModelDefinition): OpenAIChatModelRuntimeConfig;
+/**
+ * Resolves Ollama runtime config from one resolved run-context model.
+ *
+ * Use when:
+ * - task execution already has `context.model()` output
+ * - eval code wants typed Ollama provider options with a concise helper name
+ *
+ * Expects:
+ * - `model` to resolve to an Ollama-backed chat model
+ *
+ * Returns:
+ * - validated Ollama runtime config
+ */
+declare function ollamaFromRunContext(model: ModelDefinition): OllamaChatModelRuntimeConfig;
+/**
+ * Resolves OpenRouter runtime config from one resolved run-context model.
+ *
+ * Use when:
+ * - task execution already has `context.model()` output
+ * - eval code wants typed OpenRouter provider options with a concise helper name
+ *
+ * Expects:
+ * - `model` to resolve to an OpenRouter-backed chat model
+ *
+ * Returns:
+ * - validated OpenRouter runtime config
+ */
+declare function openrouterFromRunContext(model: ModelDefinition): OpenRouterChatModelRuntimeConfig;
+//#endregion
+//#region src/plugins/chat-models/telemetry.d.ts
+/**
+ * Represents one normalized chat-model tool call.
+ *
+ * Use when:
+ * - report events need tool-call level payloads that remain provider-neutral
+ *
+ * Expects:
+ * - `name` to be stable enough for aggregation and assertion checks
+ * - `args` to be JSON-serializable
+ */
+interface ChatModelToolCall {
+  /**
+   * Optional provider-assigned tool-call identifier.
+   */
+  id?: string;
+  /**
+   * Tool name.
+   */
+  name: string;
+  /**
+   * Parsed tool arguments object/value.
+   */
+  args: unknown;
+}
+/**
+ * Provider identity attached to chat-model telemetry events.
+ */
+interface ChatModelTelemetryProvider {
+  /**
+   * Provider id, for example `openai`.
+   */
+  id: string;
+  /**
+   * Optional concrete model id/name.
+   */
+  model?: string;
+}
+/**
+ * Input options for response telemetry emission.
+ */
+interface EmitChatModelResponseTelemetryOptions {
+  /**
+   * Optional case id for case-scoped telemetry events.
+   */
+  caseId?: string;
+  /**
+   * Optional response latency in milliseconds.
+   */
+  latencyMs?: number;
+  /**
+   * Optional provider identity payload.
+   */
+  provider?: ChatModelTelemetryProvider;
+  /**
+   * Raw chat-model response object from the inference library/provider.
+   */
+  response: unknown;
+}
+/**
+ * Input options for request telemetry emission.
+ */
+interface EmitChatModelRequestTelemetryOptions {
+  /**
+   * Optional case id for case-scoped telemetry events.
+   */
+  caseId?: string;
+  /**
+   * Optional request payload metadata.
+   */
+  data?: unknown;
+  /**
+   * Optional provider identity payload.
+   */
+  provider?: ChatModelTelemetryProvider;
+}
+/**
+ * Input options for error telemetry emission.
+ */
+interface EmitChatModelErrorTelemetryOptions {
+  /**
+   * Optional case id for case-scoped telemetry events.
+   */
+  caseId?: string;
+  /**
+   * Error payload emitted by the inference client/runtime.
+   */
+  error: unknown;
+  /**
+   * Optional provider identity payload.
+   */
+  provider?: ChatModelTelemetryProvider;
+}
+/**
+ * Extracts normalized tool calls from one chat-model response shape.
+ *
+ * Use when:
+ * - downstream scoring, reporting, or analysis should inspect tool call usage
+ * - provider payload differences should stay hidden behind one stable shape
+ *
+ * Returns:
+ * - normalized list of `{ id?, name, args }` tool calls
+ */
+declare function extractChatModelToolCalls(response: unknown): ChatModelToolCall[];
+/**
+ * Extracts numeric metering dimensions from one chat-model response usage block.
+ *
+ * Use when:
+ * - report events should capture usage dimensions in a modality-neutral map
+ *
+ * Returns:
+ * - numeric dimensions keyed by provider usage field names
+ */
+declare function extractMeteringDimensions(response: unknown): Record<string, number>;
+/**
+ * Emits chat-model response telemetry as reportable task events.
+ *
+ * Use when:
+ * - task code receives one chat-model response and wants standardized report events
+ * - `ToolCall*` and metering metrics should be persisted in `events.jsonl`
+ *
+ * Expects:
+ * - `context.reporterHooks?.onEvent` to be available in CLI execution paths
+ *
+ * Returns:
+ * - no return value; this is a best-effort reporting helper
+ */
+declare function emitChatModelResponseTelemetry(context: TaskRunContext, options: EmitChatModelResponseTelemetryOptions): void;
+/**
+ * Emits chat-model request telemetry as a reportable task event.
+ *
+ * Use when:
+ * - task code submits one model request and wants request-side traceability
+ *
+ * Expects:
+ * - `context.reporterHooks?.onEvent` to be available in CLI execution paths
+ */
+declare function emitChatModelRequestTelemetry(context: TaskRunContext, options: EmitChatModelRequestTelemetryOptions): void;
+/**
+ * Emits chat-model failure telemetry as a reportable task event.
+ *
+ * Use when:
+ * - one inference call fails and report artifacts should include normalized error context
+ *
+ * Expects:
+ * - `context.reporterHooks?.onEvent` to be available in CLI execution paths
+ */
+declare function emitChatModelErrorTelemetry(context: TaskRunContext, options: EmitChatModelErrorTelemetryOptions): void;
+//#endregion
 //#region src/plugins/chat-models/index.d.ts
 /**
  * Minimal inference-executor shape expected by chat model runtime callers.
@@ -12,13 +295,73 @@ interface ChatModelExecutorLike {
  */
 type ChatModelExecutorInput = string | ChatModelExecutorLike;
 /**
- * Builder input for `chatModelFrom`.
+ * Chat-model header payload accepted by executor parameters.
+ */
+type ChatModelHeaders = Record<string, string | string[]>;
+/**
+ * Runtime env context passed to model callback resolvers.
+ */
+interface ChatModelResolverContext {
+  env: Record<string, string>;
+}
+/**
+ * Value-or-callback resolver used by model runtime fields.
+ */
+type ChatModelResolverValue<TValue> = TValue | ((config: ChatModelResolverContext) => Promise<TValue> | TValue);
+/**
+ * OpenAI-specific inference executor config shape.
+ */
+interface OpenAIChatModelInferenceExecutor {
+  inferenceExecutor: 'openai';
+  apiKey?: ChatModelResolverValue<string>;
+  baseURL?: ChatModelResolverValue<string>;
+  headers?: ChatModelResolverValue<ChatModelHeaders>;
+}
+/**
+ * Ollama-specific inference executor config shape.
+ */
+interface OllamaChatModelInferenceExecutor {
+  inferenceExecutor: 'ollama';
+  baseURL?: ChatModelResolverValue<string>;
+  headers?: ChatModelResolverValue<ChatModelHeaders>;
+}
+/**
+ * OpenRouter-specific inference executor config shape.
  */
-interface ChatModelFromOptions {
+interface OpenRouterChatModelInferenceExecutor {
+  inferenceExecutor: 'openrouter';
+  apiKey?: ChatModelResolverValue<string>;
+  baseURL?: ChatModelResolverValue<string>;
+  headers?: ChatModelResolverValue<ChatModelHeaders>;
+}
+/**
+ * Generic inference executor config shape.
+ */
+interface GenericChatModelInferenceExecutor {
+  inferenceExecutor?: ChatModelExecutorInput;
+}
+/**
+ * Union of supported inference executor config shapes for `chatModelFrom`.
+ */
+type ChatModelInferenceExecutor = OpenAIChatModelInferenceExecutor | OllamaChatModelInferenceExecutor | OpenRouterChatModelInferenceExecutor | GenericChatModelInferenceExecutor;
+/**
+ * Common builder input fields for `chatModelFrom`.
+ */
+interface ChatModelFromBaseOptions {
+  /**
+   * Provider id registered through `ChatProviders`.
+   *
+   * Use when:
+   * - model runtime transport and credentials should be delegated to a named provider preset
+   *
+   * Expects:
+   * - one `ChatProviders` plugin entry to expose the same id
+   */
+  provider?: string;
   /**
    * Inference-executor id or inference-executor instance.
    */
-  inferenceExecutor: ChatModelExecutorInput;
+  inferenceExecutor?: ChatModelExecutorInput;
   /**
    * Optional explicit inference-executor id for inference-executor instances.
    *
@@ -44,16 +387,102 @@ interface ChatModelFromOptions {
    */
   parameters?: Record<string, unknown>;
 }
+/**
+ * Builder input for `chatModelFrom`.
+ */
+type ChatModelFromOptions = ChatModelInferenceExecutor & ChatModelFromBaseOptions;
 /**
  * Chat-model specific specialization of the canonical `ModelDefinition`.
  */
 type ChatModelDefinition = Omit<ModelDefinition, 'inferenceExecutor'> & {
   inferenceExecutor: ChatModelExecutorInput;
+  provider?: string;
+  runtimeResolvers?: {
+    apiKey?: ChatModelResolverValue<string>;
+    baseURL?: ChatModelResolverValue<string>;
+    headers?: ChatModelResolverValue<ChatModelHeaders>;
+  };
 };
+/**
+ * Env-key map for optional provider parameters.
+ *
+ * Use when:
+ * - provider parameter values should be read from env keys
+ * - missing keys should resolve to `undefined`
+ */
+type OptionalProviderEnvMap = Record<string, string>;
+/**
+ * Env-key map for required provider parameters.
+ *
+ * Use when:
+ * - provider parameter values must exist before model execution
+ * - missing keys should throw with key-aware error messages
+ */
+type RequiredProviderEnvMap = Record<string, string>;
+/**
+ * One provider definition consumed by chat model presets.
+ */
+interface ChatProviderDefinition {
+  /**
+   * Stable provider id referenced by `chatModelFrom({ provider })`.
+   */
+  id: string;
+  /**
+   * Inference-executor id or instance used by this provider preset.
+   */
+  inferenceExecutor: ChatModelExecutorInput;
+  /**
+   * Optional explicit inference-executor id for inference-executor instances.
+   *
+   * @default 'custom'
+   */
+  inferenceExecutorId?: string;
+  /**
+   * Optional literal provider-level parameters.
+   */
+  parameters?: Record<string, unknown>;
+  /**
+   * Optional provider parameters resolved via `envFrom`.
+   *
+   * Expects:
+   * - map key is the provider parameter name
+   * - map value is the env key name
+   */
+  optionalEnv?: OptionalProviderEnvMap;
+  /**
+   * Required provider parameters resolved via `requiredEnvFrom`.
+   *
+   * Expects:
+   * - map key is the provider parameter name
+   * - map value is the env key name
+   */
+  requiredEnv?: RequiredProviderEnvMap;
+}
+/**
+ * Builder input for `chatProviderFrom`.
+ */
+interface ChatProviderFromOptions extends ChatProviderDefinition {}
+/**
+ * Options for the built-in `ChatProviders` plugin.
+ */
+interface ChatProvidersPluginOptions {
+  /**
+   * Provider definitions to append to config.
+   */
+  providers: readonly ChatProviderDefinition[];
+  /**
+   * Optional explicit env source used for env-backed provider parameters.
+   *
+   * @default process.env
+   */
+  env?: NodeJS.ProcessEnv;
+}
 /**
  * Partial config shape needed by the chat models plugin.
  */
 interface PluginConfig {
+  env?: NodeJS.ProcessEnv;
+  chatProviders?: ChatProviderDefinition[];
   models?: ModelDefinition[];
 }
 /**
@@ -68,6 +497,14 @@ type Plugin = ConfigHookPlugin<PluginConfig>;
  * - a single model needs aliases for matrix selection or judge lookup
  */
 declare function chatModelFrom(options: ChatModelFromOptions): ChatModelDefinition;
+/**
+ * Builds one normalized chat provider definition.
+ *
+ * Use when:
+ * - one provider preset should be reused across multiple chat models
+ * - provider configuration should support required/optional env-backed parameters
+ */
+declare function chatProviderFrom(options: ChatProviderFromOptions): ChatProviderDefinition;
 /**
  * Options for the built-in `ChatModels` plugin.
  */
@@ -75,8 +512,30 @@ interface ChatModelsPluginOptions {
   /**
    * Chat model definitions to append to config.
    */
-  models: readonly ModelDefinition[];
+  models: readonly ChatModelDefinition[];
 }
+/**
+ * Creates a run-matrix `model` axis from configured chat model names.
+ *
+ * Use when:
+ * - run matrix should iterate over explicit chat model ids/aliases
+ * - project configs want a concise model-axis helper
+ *
+ * Expects:
+ * - each provided name to match a configured model id or alias at runtime
+ *
+ * Returns:
+ * - matrix axis object compatible with `runMatrix.extend/override`
+ */
+declare function chatModelMatrix(...names: string[]): MatrixDefinition;
+/**
+ * Built-in chat providers plugin that contributes provider presets to config.
+ *
+ * Use when:
+ * - provider runtime config should be centralized and reusable
+ * - provider parameters should be resolved from env via `envFrom`/`requiredEnvFrom`
+ */
+declare function ChatProviders(options: ChatProvidersPluginOptions): Plugin;
 /**
  * Built-in chat models plugin that contributes model definitions to vieval config.
  *
@@ -86,5 +545,5 @@ interface ChatModelsPluginOptions {
  */
 declare function ChatModels(options: ChatModelsPluginOptions): Plugin;
 //#endregion
-export { ChatModelDefinition, ChatModelExecutorInput, ChatModelExecutorLike, ChatModelFromOptions, ChatModels, ChatModelsPluginOptions, Plugin, PluginConfig, chatModelFrom };
+export { ChatModelDefinition, ChatModelExecutorInput, ChatModelExecutorLike, ChatModelFromBaseOptions, ChatModelFromOptions, ChatModelHeaders, ChatModelInferenceExecutor, ChatModelResolverContext, ChatModelResolverValue, ChatModelRuntimeConfig, ChatModelTelemetryProvider, ChatModelToolCall, ChatModels, ChatModelsPluginOptions, ChatProviderDefinition, ChatProviderFromOptions, ChatProviders, ChatProvidersPluginOptions, EmitChatModelErrorTelemetryOptions, EmitChatModelRequestTelemetryOptions, EmitChatModelResponseTelemetryOptions, GenericChatModelInferenceExecutor, OllamaChatModelInferenceExecutor, OllamaChatModelRuntimeConfig, OpenAIChatModelInferenceExecutor, OpenAIChatModelRuntimeConfig, OpenRouterChatModelInferenceExecutor, OpenRouterChatModelRuntimeConfig, OptionalProviderEnvMap, Plugin, PluginConfig, RequiredProviderEnvMap, chatModelFrom, chatModelMatrix, chatProviderFrom, emitChatModelErrorTelemetry, emitChatModelRequestTelemetry, emitChatModelResponseTelemetry, extractChatModelToolCalls, extractMeteringDimensions, ollamaFromRunContext, openaiFromRunContext, openrouterFromRunContext, toChatModelRuntimeConfig };
 //# sourceMappingURL=index.d.mts.map