scorecard-ai 1.0.0-alpha.8 → 1.0.0

package/src/resources/metrics.ts

@@ -0,0 +1,525 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { APIPromise } from '../core/api-promise';
+import { RequestOptions } from '../internal/request-options';
+import { path } from '../internal/utils/path';
+
+export class Metrics extends APIResource {
+  /**
+   * Create a new Metric for evaluating system outputs.
+   *
+   * @example
+   * ```ts
+   * const metric = await client.metrics.create('314', {
+   *   evalType: 'ai',
+   *   name: 'Response Accuracy',
+   *   outputType: 'boolean',
+   *   promptTemplate:
+   *     'Please evaluate if the following response is factually accurate: {{outputs.response}}',
+   *   description:
+   *     'Evaluates if the response is factually accurate',
+   *   evalModelName: 'gpt-4o',
+   *   guidelines:
+   *     'Check if the response contains factually correct information',
+   *   temperature: 0.1,
+   * });
+   * ```
+   */
+  create(projectID: string, body: MetricCreateParams, options?: RequestOptions): APIPromise<Metric> {
+    return this._client.post(path`/projects/${projectID}/metrics`, { body, ...options });
+  }
+}
+
+/**
+ * A Metric defines how to evaluate system outputs against expected results.
+ */
+export type Metric =
+  | Metric.AIIntMetric
+  | Metric.HumanIntMetric
+  | Metric.HeuristicIntMetric
+  | Metric.AIBooleanMetric
+  | Metric.HumanBooleanMetric
+  | Metric.HeuristicBooleanMetric;
+
+export namespace Metric {
+  /**
+   * A Metric with AI evaluation and integer output.
+   */
+  export interface AIIntMetric {
+    /**
+     * The ID of the Metric.
+     */
+    id: string;
+
+    /**
+     * The description of the Metric.
+     */
+    description: string | null;
+
+    /**
+     * The AI model to use for evaluation.
+     */
+    evalModelName: string;
+
+    /**
+     * AI-based evaluation type.
+     */
+    evalType: 'ai';
+
+    /**
+     * Guidelines for AI evaluation on how to score the metric.
+     */
+    guidelines: string | null;
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Integer output type.
+     */
+    outputType: 'int';
+
+    /**
+     * The threshold for determining pass/fail from integer scores (1-5).
+     */
+    passingThreshold: number;
+
+    /**
+     * The complete prompt template for AI evaluation. Should include placeholders for
+     * dynamic content.
+     */
+    promptTemplate: string;
+
+    /**
+     * The temperature for AI evaluation (0-2).
+     */
+    temperature: number;
+  }
+
+  /**
+   * A Metric with human evaluation and integer output.
+   */
+  export interface HumanIntMetric {
+    /**
+     * The ID of the Metric.
+     */
+    id: string;
+
+    /**
+     * The description of the Metric.
+     */
+    description: string | null;
+
+    /**
+     * Human-based evaluation type.
+     */
+    evalType: 'human';
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Integer output type.
+     */
+    outputType: 'int';
+
+    /**
+     * The threshold for determining pass/fail from integer scores (1-5).
+     */
+    passingThreshold: number;
+
+    /**
+     * Guidelines for human evaluators.
+     */
+    guidelines?: string;
+  }
+
+  /**
+   * A Metric with heuristic evaluation and integer output.
+   */
+  export interface HeuristicIntMetric {
+    /**
+     * The ID of the Metric.
+     */
+    id: string;
+
+    /**
+     * The description of the Metric.
+     */
+    description: string | null;
+
+    /**
+     * Heuristic-based evaluation type.
+     */
+    evalType: 'heuristic';
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Integer output type.
+     */
+    outputType: 'int';
+
+    /**
+     * The threshold for determining pass/fail from integer scores (1-5).
+     */
+    passingThreshold: number;
+
+    /**
+     * Optional guidelines for heuristic evaluation logic.
+     */
+    guidelines?: string;
+  }
+
+  /**
+   * A Metric with AI evaluation and boolean output.
+   */
+  export interface AIBooleanMetric {
+    /**
+     * The ID of the Metric.
+     */
+    id: string;
+
+    /**
+     * The description of the Metric.
+     */
+    description: string | null;
+
+    /**
+     * The AI model to use for evaluation.
+     */
+    evalModelName: string;
+
+    /**
+     * AI-based evaluation type.
+     */
+    evalType: 'ai';
+
+    /**
+     * Guidelines for AI evaluation on how to score the metric.
+     */
+    guidelines: string | null;
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Boolean output type.
+     */
+    outputType: 'boolean';
+
+    /**
+     * The complete prompt template for AI evaluation. Should include placeholders for
+     * dynamic content.
+     */
+    promptTemplate: string;
+
+    /**
+     * The temperature for AI evaluation (0-2).
+     */
+    temperature: number;
+  }
+
+  /**
+   * A Metric with human evaluation and boolean output.
+   */
+  export interface HumanBooleanMetric {
+    /**
+     * The ID of the Metric.
+     */
+    id: string;
+
+    /**
+     * The description of the Metric.
+     */
+    description: string | null;
+
+    /**
+     * Human-based evaluation type.
+     */
+    evalType: 'human';
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Boolean output type.
+     */
+    outputType: 'boolean';
+
+    /**
+     * Guidelines for human evaluators.
+     */
+    guidelines?: string;
+  }
+
+  /**
+   * A Metric with heuristic evaluation and boolean output.
+   */
+  export interface HeuristicBooleanMetric {
+    /**
+     * The ID of the Metric.
+     */
+    id: string;
+
+    /**
+     * The description of the Metric.
+     */
+    description: string | null;
+
+    /**
+     * Heuristic-based evaluation type.
+     */
+    evalType: 'heuristic';
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Boolean output type.
+     */
+    outputType: 'boolean';
+
+    /**
+     * Optional guidelines for heuristic evaluation logic.
+     */
+    guidelines?: string;
+  }
+}
+
+export type MetricCreateParams =
+  | MetricCreateParams.AIIntMetric
+  | MetricCreateParams.HumanIntMetric
+  | MetricCreateParams.HeuristicIntMetric
+  | MetricCreateParams.AIBooleanMetric
+  | MetricCreateParams.HumanBooleanMetric
+  | MetricCreateParams.HeuristicBooleanMetric;
+
+export declare namespace MetricCreateParams {
+  export interface AIIntMetric {
+    /**
+     * AI-based evaluation type.
+     */
+    evalType: 'ai';
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Integer output type.
+     */
+    outputType: 'int';
+
+    /**
+     * The complete prompt template for AI evaluation. Should include placeholders for
+     * dynamic content.
+     */
+    promptTemplate: string;
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * The AI model to use for evaluation.
+     */
+    evalModelName?: string;
+
+    /**
+     * Guidelines for AI evaluation on how to score the metric.
+     */
+    guidelines?: string | null;
+
+    /**
+     * The threshold for determining pass/fail from integer scores (1-5).
+     */
+    passingThreshold?: number;
+
+    /**
+     * The temperature for AI evaluation (0-2).
+     */
+    temperature?: number;
+  }
+
+  export interface HumanIntMetric {
+    /**
+     * Human-based evaluation type.
+     */
+    evalType: 'human';
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Integer output type.
+     */
+    outputType: 'int';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * Guidelines for human evaluators.
+     */
+    guidelines?: string;
+
+    /**
+     * The threshold for determining pass/fail from integer scores (1-5).
+     */
+    passingThreshold?: number;
+  }
+
+  export interface HeuristicIntMetric {
+    /**
+     * Heuristic-based evaluation type.
+     */
+    evalType: 'heuristic';
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Integer output type.
+     */
+    outputType: 'int';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * Optional guidelines for heuristic evaluation logic.
+     */
+    guidelines?: string;
+
+    /**
+     * The threshold for determining pass/fail from integer scores (1-5).
+     */
+    passingThreshold?: number;
+  }
+
+  export interface AIBooleanMetric {
+    /**
+     * AI-based evaluation type.
+     */
+    evalType: 'ai';
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Boolean output type.
+     */
+    outputType: 'boolean';
+
+    /**
+     * The complete prompt template for AI evaluation. Should include placeholders for
+     * dynamic content.
+     */
+    promptTemplate: string;
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * The AI model to use for evaluation.
+     */
+    evalModelName?: string;
+
+    /**
+     * Guidelines for AI evaluation on how to score the metric.
+     */
+    guidelines?: string | null;
+
+    /**
+     * The temperature for AI evaluation (0-2).
+     */
+    temperature?: number;
+  }
+
+  export interface HumanBooleanMetric {
+    /**
+     * Human-based evaluation type.
+     */
+    evalType: 'human';
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Boolean output type.
+     */
+    outputType: 'boolean';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * Guidelines for human evaluators.
+     */
+    guidelines?: string;
+  }
+
+  export interface HeuristicBooleanMetric {
+    /**
+     * Heuristic-based evaluation type.
+     */
+    evalType: 'heuristic';
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Boolean output type.
+     */
+    outputType: 'boolean';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * Optional guidelines for heuristic evaluation logic.
+     */
+    guidelines?: string;
+  }
+}
+
+export declare namespace Metrics {
+  export { type Metric as Metric, type MetricCreateParams as MetricCreateParams };
+}

package/src/resources/runs.ts

@@ -13,8 +13,8 @@ export class Runs extends APIResource {
    * ```ts
    * const run = await client.runs.create('314', {
    *   metricIds: ['789', '101'],
+   *   systemVersionId: '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
    *   testsetId: '246',
-   *   systemConfigId: '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
    * });
    * ```
    */
@@ -52,12 +52,12 @@ export interface Run {
   /**
    * The ID of the Testset this Run is testing.
    */
-  testsetId: string;
+  testsetId: string | null;
 
   /**
-   * The ID of the system configuration this Run is using.
+   * The ID of the system version this Run is using.
    */
-  systemConfigId?: string;
+  systemVersionId?: string;
 }
 
 export interface RunCreateParams {
@@ -67,14 +67,14 @@ export interface RunCreateParams {
   metricIds: Array<string>;
 
   /**
-   * The ID of the Testset this Run is testing.
+   * The ID of the system version this Run is using.
    */
-  testsetId: string;
+  systemVersionId?: string;
 
   /**
-   * The ID of the system configuration this Run is using.
+   * The ID of the Testset this Run is testing.
    */
-  systemConfigId?: string;
+  testsetId?: string | null;
 }
 
 export declare namespace Runs {

package/src/resources/systems/index.ts

@@ -0,0 +1,18 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+export {
+  Systems,
+  type System,
+  type SystemDeleteResponse,
+  type SystemCreateParams,
+  type SystemUpdateParams,
+  type SystemListParams,
+  type SystemsPaginatedResponse,
+} from './systems';
+export {
+  Versions,
+  type SystemVersion,
+  type VersionCreateParams,
+  type VersionListParams,
+  type SystemVersionsPaginatedResponse,
+} from './versions';