scorecard-ai 2.1.0 → 2.2.0

package/src/resources/metrics.ts

@@ -2,6 +2,7 @@
 
 import { APIResource } from '../core/resource';
 import { APIPromise } from '../core/api-promise';
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../core/pagination';
 import { RequestOptions } from '../internal/request-options';
 import { path } from '../internal/utils/path';
 
@@ -49,8 +50,45 @@ export class Metrics extends APIResource {
   update(metricID: string, body: MetricUpdateParams, options?: RequestOptions): APIPromise<Metric> {
     return this._client.patch(path`/metrics/${metricID}`, { body, ...options });
   }
+
+  /**
+   * List Metrics configured for the specified Project. Metrics are returned in
+   * reverse chronological order.
+   *
+   * @example
+   * ```ts
+   * // Automatically fetches more pages as needed.
+   * for await (const metric of client.metrics.list('314')) {
+   *   // ...
+   * }
+   * ```
+   */
+  list(
+    projectID: string,
+    query: MetricListParams | null | undefined = {},
+    options?: RequestOptions,
+  ): PagePromise<MetricsPaginatedResponse, Metric> {
+    return this._client.getAPIList(path`/projects/${projectID}/metrics`, PaginatedResponse<Metric>, {
+      query,
+      ...options,
+    });
+  }
+
+  /**
+   * Retrieve a specific Metric by ID.
+   *
+   * @example
+   * ```ts
+   * const metric = await client.metrics.get('321');
+   * ```
+   */
+  get(metricID: string, options?: RequestOptions): APIPromise<Metric> {
+    return this._client.get(path`/metrics/${metricID}`, options);
+  }
 }
 
+export type MetricsPaginatedResponse = PaginatedResponse<Metric>;
+
 /**
  * A Metric defines how to evaluate system outputs against expected results.
  */
@@ -58,6 +96,9 @@ export type Metric =
   | Metric.AIIntMetric
   | Metric.HumanIntMetric
   | Metric.HeuristicIntMetric
+  | Metric.AIFloatMetric
+  | Metric.HumanFloatMetric
+  | Metric.HeuristicFloatMetric
   | Metric.AIBooleanMetric
   | Metric.HumanBooleanMetric
   | Metric.HeuristicBooleanMetric;
@@ -90,7 +131,7 @@ export namespace Metric {
     /**
      * Guidelines for AI evaluation on how to score the metric.
      */
-    guidelines: string | null;
+    guidelines: string;
 
     /**
      * The name of the Metric.
@@ -138,6 +179,11 @@ export namespace Metric {
      */
     evalType: 'human';
 
+    /**
+     * Guidelines for human evaluators.
+     */
+    guidelines: string;
+
     /**
      * The name of the Metric.
      */
@@ -152,11 +198,6 @@ export namespace Metric {
      * The threshold for determining pass/fail from integer scores (1-5).
      */
     passingThreshold: number;
-
-    /**
-     * Guidelines for human evaluators.
-     */
-    guidelines?: string;
   }
 
   /**
@@ -178,6 +219,11 @@ export namespace Metric {
      */
     evalType: 'heuristic';
 
+    /**
+     * Guidelines for heuristic evaluation logic.
+     */
+    guidelines: string;
+
     /**
      * The name of the Metric.
      */
@@ -192,11 +238,142 @@ export namespace Metric {
      * The threshold for determining pass/fail from integer scores (1-5).
      */
     passingThreshold: number;
+  }
 
+  /**
+   * A Metric with AI evaluation and float output.
+   */
+  export interface AIFloatMetric {
     /**
-     * Optional guidelines for heuristic evaluation logic.
+     * The ID of the Metric.
      */
-    guidelines?: string;
+    id: string;
+
+    /**
+     * The description of the Metric.
+     */
+    description: string | null;
+
+    /**
+     * The AI model to use for evaluation.
+     */
+    evalModelName: string;
+
+    /**
+     * AI-based evaluation type.
+     */
+    evalType: 'ai';
+
+    /**
+     * Guidelines for AI evaluation on how to score the metric.
+     */
+    guidelines: string;
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Float output type (0-1).
+     */
+    outputType: 'float';
+
+    /**
+     * Threshold for determining pass/fail from float scores (0.0-1.0).
+     */
+    passingThreshold: number;
+
+    /**
+     * The complete prompt template for AI evaluation. Should include placeholders for
+     * dynamic content.
+     */
+    promptTemplate: string;
+
+    /**
+     * The temperature for AI evaluation (0-2).
+     */
+    temperature: number;
+  }
+
+  /**
+   * A Metric with human evaluation and float output.
+   */
+  export interface HumanFloatMetric {
+    /**
+     * The ID of the Metric.
+     */
+    id: string;
+
+    /**
+     * The description of the Metric.
+     */
+    description: string | null;
+
+    /**
+     * Human-based evaluation type.
+     */
+    evalType: 'human';
+
+    /**
+     * Guidelines for human evaluators.
+     */
+    guidelines: string;
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Float output type (0-1).
+     */
+    outputType: 'float';
+
+    /**
+     * Threshold for determining pass/fail from float scores (0.0-1.0).
+     */
+    passingThreshold: number;
+  }
+
+  /**
+   * A Metric with heuristic evaluation and float output.
+   */
+  export interface HeuristicFloatMetric {
+    /**
+     * The ID of the Metric.
+     */
+    id: string;
+
+    /**
+     * The description of the Metric.
+     */
+    description: string | null;
+
+    /**
+     * Heuristic-based evaluation type.
+     */
+    evalType: 'heuristic';
+
+    /**
+     * Guidelines for heuristic evaluation logic.
+     */
+    guidelines: string;
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Float output type (0-1).
+     */
+    outputType: 'float';
+
+    /**
+     * Threshold for determining pass/fail from float scores (0.0-1.0).
+     */
+    passingThreshold: number;
   }
 
   /**
@@ -226,7 +403,7 @@ export namespace Metric {
     /**
      * Guidelines for AI evaluation on how to score the metric.
      */
-    guidelines: string | null;
+    guidelines: string;
 
     /**
      * The name of the Metric.
@@ -269,6 +446,11 @@ export namespace Metric {
      */
     evalType: 'human';
 
+    /**
+     * Guidelines for human evaluators.
+     */
+    guidelines: string;
+
     /**
      * The name of the Metric.
      */
@@ -278,11 +460,6 @@ export namespace Metric {
      * Boolean output type.
      */
     outputType: 'boolean';
-
-    /**
-     * Guidelines for human evaluators.
-     */
-    guidelines?: string;
   }
 
   /**
@@ -304,6 +481,11 @@ export namespace Metric {
      */
     evalType: 'heuristic';
 
+    /**
+     * Guidelines for heuristic evaluation logic.
+     */
+    guidelines: string;
+
     /**
      * The name of the Metric.
      */
@@ -313,11 +495,6 @@ export namespace Metric {
      * Boolean output type.
      */
     outputType: 'boolean';
-
-    /**
-     * Optional guidelines for heuristic evaluation logic.
-     */
-    guidelines?: string;
   }
 }
 
@@ -325,6 +502,9 @@ export type MetricCreateParams =
   | MetricCreateParams.AIIntMetric
   | MetricCreateParams.HumanIntMetric
   | MetricCreateParams.HeuristicIntMetric
+  | MetricCreateParams.AIFloatMetric
+  | MetricCreateParams.HumanFloatMetric
+  | MetricCreateParams.HeuristicFloatMetric
   | MetricCreateParams.AIBooleanMetric
   | MetricCreateParams.HumanBooleanMetric
   | MetricCreateParams.HeuristicBooleanMetric;
@@ -365,7 +545,7 @@ export declare namespace MetricCreateParams {
     /**
      * Guidelines for AI evaluation on how to score the metric.
      */
-    guidelines?: string | null;
+    guidelines?: string;
 
     /**
      * The threshold for determining pass/fail from integer scores (1-5).
@@ -432,7 +612,7 @@ export declare namespace MetricCreateParams {
     description?: string | null;
 
     /**
-     * Optional guidelines for heuristic evaluation logic.
+     * Guidelines for heuristic evaluation logic.
      */
     guidelines?: string;
 
@@ -442,6 +622,118 @@ export declare namespace MetricCreateParams {
     passingThreshold?: number;
   }
 
+  export interface AIFloatMetric {
+    /**
+     * AI-based evaluation type.
+     */
+    evalType: 'ai';
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Float output type (0-1).
+     */
+    outputType: 'float';
+
+    /**
+     * The complete prompt template for AI evaluation. Should include placeholders for
+     * dynamic content.
+     */
+    promptTemplate: string;
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * The AI model to use for evaluation.
+     */
+    evalModelName?: string;
+
+    /**
+     * Guidelines for AI evaluation on how to score the metric.
+     */
+    guidelines?: string;
+
+    /**
+     * Threshold for determining pass/fail from float scores (0.0-1.0).
+     */
+    passingThreshold?: number;
+
+    /**
+     * The temperature for AI evaluation (0-2).
+     */
+    temperature?: number;
+  }
+
+  export interface HumanFloatMetric {
+    /**
+     * Human-based evaluation type.
+     */
+    evalType: 'human';
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Float output type (0-1).
+     */
+    outputType: 'float';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * Guidelines for human evaluators.
+     */
+    guidelines?: string;
+
+    /**
+     * Threshold for determining pass/fail from float scores (0.0-1.0).
+     */
+    passingThreshold?: number;
+  }
+
+  export interface HeuristicFloatMetric {
+    /**
+     * Heuristic-based evaluation type.
+     */
+    evalType: 'heuristic';
+
+    /**
+     * The name of the Metric.
+     */
+    name: string;
+
+    /**
+     * Float output type (0-1).
+     */
+    outputType: 'float';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * Guidelines for heuristic evaluation logic.
+     */
+    guidelines?: string;
+
+    /**
+     * Threshold for determining pass/fail from float scores (0.0-1.0).
+     */
+    passingThreshold?: number;
+  }
+
   export interface AIBooleanMetric {
     /**
      * AI-based evaluation type.
@@ -477,7 +769,7 @@ export declare namespace MetricCreateParams {
     /**
      * Guidelines for AI evaluation on how to score the metric.
      */
-    guidelines?: string | null;
+    guidelines?: string;
 
     /**
      * The temperature for AI evaluation (0-2).
@@ -534,7 +826,7 @@ export declare namespace MetricCreateParams {
     description?: string | null;
 
     /**
-     * Optional guidelines for heuristic evaluation logic.
+     * Guidelines for heuristic evaluation logic.
      */
     guidelines?: string;
   }
@@ -544,6 +836,9 @@ export type MetricUpdateParams =
   | MetricUpdateParams.AIIntMetric
   | MetricUpdateParams.HumanIntMetric
   | MetricUpdateParams.HeuristicIntMetric
+  | MetricUpdateParams.AIFloatMetric
+  | MetricUpdateParams.HumanFloatMetric
+  | MetricUpdateParams.HeuristicFloatMetric
   | MetricUpdateParams.AIBooleanMetric
   | MetricUpdateParams.HumanBooleanMetric
   | MetricUpdateParams.HeuristicBooleanMetric;
@@ -573,7 +868,7 @@ export declare namespace MetricUpdateParams {
     /**
      * Guidelines for AI evaluation on how to score the metric.
      */
-    guidelines?: string | null;
+    guidelines?: string;
 
     /**
      * The name of the Metric.
@@ -646,7 +941,7 @@ export declare namespace MetricUpdateParams {
     description?: string | null;
 
     /**
-     * Optional guidelines for heuristic evaluation logic.
+     * Guidelines for heuristic evaluation logic.
      */
     guidelines?: string;
 
@@ -661,6 +956,118 @@ export declare namespace MetricUpdateParams {
     passingThreshold?: number;
   }
 
+  export interface AIFloatMetric {
+    /**
+     * AI-based evaluation type.
+     */
+    evalType: 'ai';
+
+    /**
+     * Float output type (0-1).
+     */
+    outputType: 'float';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * The AI model to use for evaluation.
+     */
+    evalModelName?: string;
+
+    /**
+     * Guidelines for AI evaluation on how to score the metric.
+     */
+    guidelines?: string;
+
+    /**
+     * The name of the Metric.
+     */
+    name?: string;
+
+    /**
+     * Threshold for determining pass/fail from float scores (0.0-1.0).
+     */
+    passingThreshold?: number;
+
+    /**
+     * The complete prompt template for AI evaluation. Should include placeholders for
+     * dynamic content.
+     */
+    promptTemplate?: string;
+
+    /**
+     * The temperature for AI evaluation (0-2).
+     */
+    temperature?: number;
+  }
+
+  export interface HumanFloatMetric {
+    /**
+     * Human-based evaluation type.
+     */
+    evalType: 'human';
+
+    /**
+     * Float output type (0-1).
+     */
+    outputType: 'float';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * Guidelines for human evaluators.
+     */
+    guidelines?: string;
+
+    /**
+     * The name of the Metric.
+     */
+    name?: string;
+
+    /**
+     * Threshold for determining pass/fail from float scores (0.0-1.0).
+     */
+    passingThreshold?: number;
+  }
+
+  export interface HeuristicFloatMetric {
+    /**
+     * Heuristic-based evaluation type.
+     */
+    evalType: 'heuristic';
+
+    /**
+     * Float output type (0-1).
+     */
+    outputType: 'float';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * Guidelines for heuristic evaluation logic.
+     */
+    guidelines?: string;
+
+    /**
+     * The name of the Metric.
+     */
+    name?: string;
+
+    /**
+     * Threshold for determining pass/fail from float scores (0.0-1.0).
+     */
+    passingThreshold?: number;
+  }
+
   export interface AIBooleanMetric {
     /**
      * AI-based evaluation type.
@@ -685,7 +1092,7 @@ export declare namespace MetricUpdateParams {
     /**
      * Guidelines for AI evaluation on how to score the metric.
      */
-    guidelines?: string | null;
+    guidelines?: string;
 
     /**
      * The name of the Metric.
@@ -748,7 +1155,7 @@ export declare namespace MetricUpdateParams {
     description?: string | null;
 
     /**
-     * Optional guidelines for heuristic evaluation logic.
+     * Guidelines for heuristic evaluation logic.
      */
     guidelines?: string;
 
@@ -759,10 +1166,14 @@ export declare namespace MetricUpdateParams {
   }
 }
 
+export interface MetricListParams extends PaginatedResponseParams {}
+
 export declare namespace Metrics {
   export {
     type Metric as Metric,
+    type MetricsPaginatedResponse as MetricsPaginatedResponse,
     type MetricCreateParams as MetricCreateParams,
     type MetricUpdateParams as MetricUpdateParams,
+    type MetricListParams as MetricListParams,
   };
 }

package/src/resources/records.ts

@@ -1,7 +1,9 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 import { APIResource } from '../core/resource';
+import * as ScoresAPI from './scores';
 import { APIPromise } from '../core/api-promise';
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../core/pagination';
 import { RequestOptions } from '../internal/request-options';
 import { path } from '../internal/utils/path';
 
@@ -24,8 +26,35 @@ export class Records extends APIResource {
   create(runID: string, body: RecordCreateParams, options?: RequestOptions): APIPromise<Record> {
     return this._client.post(path`/runs/${runID}/records`, { body, ...options });
   }
+
+  /**
+   * Retrieve a paginated list of Records for a Run, including all scores for each
+   * record.
+   *
+   * @example
+   * ```ts
+   * // Automatically fetches more pages as needed.
+   * for await (const recordListResponse of client.records.list(
+   *   '135',
+   * )) {
+   *   // ...
+   * }
+   * ```
+   */
+  list(
+    runID: string,
+    query: RecordListParams | null | undefined = {},
+    options?: RequestOptions,
+  ): PagePromise<RecordListResponsesPaginatedResponse, RecordListResponse> {
+    return this._client.getAPIList(path`/runs/${runID}/records`, PaginatedResponse<RecordListResponse>, {
+      query,
+      ...options,
+    });
+  }
 }
 
+export type RecordListResponsesPaginatedResponse = PaginatedResponse<RecordListResponse>;
+
 /**
  * A record of a system execution in the Scorecard system.
  */
@@ -62,6 +91,16 @@ export interface Record {
   testcaseId?: string;
 }
 
+/**
+ * A record with all its associated scores.
+ */
+export interface RecordListResponse extends Record {
+  /**
+   * All scores associated with this record.
+   */
+  scores: Array<ScoresAPI.Score>;
+}
+
 export interface RecordCreateParams {
   /**
    * The expected outputs for the Testcase.
@@ -85,6 +124,14 @@ export interface RecordCreateParams {
   testcaseId?: string;
 }
 
+export interface RecordListParams extends PaginatedResponseParams {}
+
 export declare namespace Records {
-  export { type Record as Record, type RecordCreateParams as RecordCreateParams };
+  export {
+    type Record as Record,
+    type RecordListResponse as RecordListResponse,
+    type RecordListResponsesPaginatedResponse as RecordListResponsesPaginatedResponse,
+    type RecordCreateParams as RecordCreateParams,
+    type RecordListParams as RecordListParams,
+  };
 }