scorecard-ai 1.0.0 → 1.2.0

package/src/resources/metrics.ts

@@ -7,7 +7,8 @@ import { path } from '../internal/utils/path';
 
 export class Metrics extends APIResource {
   /**
-   * Create a new Metric for evaluating system outputs.
+   * Create a new Metric for evaluating system outputs. The structure of a metric
+   * depends on the evalType and outputType of the metric.
    *
    * @example
    * ```ts
@@ -29,6 +30,25 @@ export class Metrics extends APIResource {
   create(projectID: string, body: MetricCreateParams, options?: RequestOptions): APIPromise<Metric> {
     return this._client.post(path`/projects/${projectID}/metrics`, { body, ...options });
   }
+
+  /**
+   * Update an existing Metric. You must specify the evalType and outputType of the
+   * metric. The structure of a metric depends on the evalType and outputType of the
+   * metric.
+   *
+   * @example
+   * ```ts
+   * const metric = await client.metrics.update('321', {
+   *   evalType: 'ai',
+   *   outputType: 'boolean',
+   *   promptTemplate:
+   *     'Using the following guidelines, evaluate the response: {{ guidelines }}\n\nResponse: {{ outputs.response }}\n\nIdeal answer: {{ expected.idealResponse }}',
+   * });
+   * ```
+   */
+  update(metricID: string, body: MetricUpdateParams, options?: RequestOptions): APIPromise<Metric> {
+    return this._client.patch(path`/metrics/${metricID}`, { body, ...options });
+  }
 }
 
 /**
@@ -520,6 +540,229 @@ export declare namespace MetricCreateParams {
   }
 }
 
+export type MetricUpdateParams =
+  | MetricUpdateParams.AIIntMetric
+  | MetricUpdateParams.HumanIntMetric
+  | MetricUpdateParams.HeuristicIntMetric
+  | MetricUpdateParams.AIBooleanMetric
+  | MetricUpdateParams.HumanBooleanMetric
+  | MetricUpdateParams.HeuristicBooleanMetric;
+
+export declare namespace MetricUpdateParams {
+  export interface AIIntMetric {
+    /**
+     * AI-based evaluation type.
+     */
+    evalType: 'ai';
+
+    /**
+     * Integer output type.
+     */
+    outputType: 'int';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * The AI model to use for evaluation.
+     */
+    evalModelName?: string;
+
+    /**
+     * Guidelines for AI evaluation on how to score the metric.
+     */
+    guidelines?: string | null;
+
+    /**
+     * The name of the Metric.
+     */
+    name?: string;
+
+    /**
+     * The threshold for determining pass/fail from integer scores (1-5).
+     */
+    passingThreshold?: number;
+
+    /**
+     * The complete prompt template for AI evaluation. Should include placeholders for
+     * dynamic content.
+     */
+    promptTemplate?: string;
+
+    /**
+     * The temperature for AI evaluation (0-2).
+     */
+    temperature?: number;
+  }
+
+  export interface HumanIntMetric {
+    /**
+     * Human-based evaluation type.
+     */
+    evalType: 'human';
+
+    /**
+     * Integer output type.
+     */
+    outputType: 'int';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * Guidelines for human evaluators.
+     */
+    guidelines?: string;
+
+    /**
+     * The name of the Metric.
+     */
+    name?: string;
+
+    /**
+     * The threshold for determining pass/fail from integer scores (1-5).
+     */
+    passingThreshold?: number;
+  }
+
+  export interface HeuristicIntMetric {
+    /**
+     * Heuristic-based evaluation type.
+     */
+    evalType: 'heuristic';
+
+    /**
+     * Integer output type.
+     */
+    outputType: 'int';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * Optional guidelines for heuristic evaluation logic.
+     */
+    guidelines?: string;
+
+    /**
+     * The name of the Metric.
+     */
+    name?: string;
+
+    /**
+     * The threshold for determining pass/fail from integer scores (1-5).
+     */
+    passingThreshold?: number;
+  }
+
+  export interface AIBooleanMetric {
+    /**
+     * AI-based evaluation type.
+     */
+    evalType: 'ai';
+
+    /**
+     * Boolean output type.
+     */
+    outputType: 'boolean';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * The AI model to use for evaluation.
+     */
+    evalModelName?: string;
+
+    /**
+     * Guidelines for AI evaluation on how to score the metric.
+     */
+    guidelines?: string | null;
+
+    /**
+     * The name of the Metric.
+     */
+    name?: string;
+
+    /**
+     * The complete prompt template for AI evaluation. Should include placeholders for
+     * dynamic content.
+     */
+    promptTemplate?: string;
+
+    /**
+     * The temperature for AI evaluation (0-2).
+     */
+    temperature?: number;
+  }
+
+  export interface HumanBooleanMetric {
+    /**
+     * Human-based evaluation type.
+     */
+    evalType: 'human';
+
+    /**
+     * Boolean output type.
+     */
+    outputType: 'boolean';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * Guidelines for human evaluators.
+     */
+    guidelines?: string;
+
+    /**
+     * The name of the Metric.
+     */
+    name?: string;
+  }
+
+  export interface HeuristicBooleanMetric {
+    /**
+     * Heuristic-based evaluation type.
+     */
+    evalType: 'heuristic';
+
+    /**
+     * Boolean output type.
+     */
+    outputType: 'boolean';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * Optional guidelines for heuristic evaluation logic.
+     */
+    guidelines?: string;
+
+    /**
+     * The name of the Metric.
+     */
+    name?: string;
+  }
+}
+
 export declare namespace Metrics {
-  export { type Metric as Metric, type MetricCreateParams as MetricCreateParams };
+  export {
+    type Metric as Metric,
+    type MetricCreateParams as MetricCreateParams,
+    type MetricUpdateParams as MetricUpdateParams,
+  };
 }

package/src/resources/records.ts

@@ -2,7 +2,6 @@
 
 import { APIResource } from '../core/resource';
 import { APIPromise } from '../core/api-promise';
-import { type Record as BuiltinRecord } from '../internal/builtin-types';
 import { RequestOptions } from '../internal/request-options';
 import { path } from '../internal/utils/path';
 
@@ -39,18 +38,18 @@ export interface Record {
   /**
    * The expected outputs for the Testcase.
    */
-  expected: BuiltinRecord<string, unknown>;
+  expected: { [key: string]: unknown };
 
   /**
    * The actual inputs sent to the system, which should match the system's input
    * schema.
    */
-  inputs: BuiltinRecord<string, unknown>;
+  inputs: { [key: string]: unknown };
 
   /**
    * The actual outputs from the system.
    */
-  outputs: BuiltinRecord<string, unknown>;
+  outputs: { [key: string]: unknown };
 
   /**
    * The ID of the Run containing this Record.
@@ -67,18 +66,18 @@ export interface RecordCreateParams {
   /**
    * The expected outputs for the Testcase.
    */
-  expected: BuiltinRecord<string, unknown>;
+  expected: { [key: string]: unknown };
 
   /**
    * The actual inputs sent to the system, which should match the system's input
    * schema.
    */
-  inputs: BuiltinRecord<string, unknown>;
+  inputs: { [key: string]: unknown };
 
   /**
    * The actual outputs from the system.
    */
-  outputs: BuiltinRecord<string, unknown>;
+  outputs: { [key: string]: unknown };
 
   /**
    * The ID of the Testcase.

package/src/resources/scores.ts

@@ -56,7 +56,7 @@ export interface Score {
    * the output schema defined by the associated MetricConfig. If it doesn't,
    * validation errors will be captured in the `validationErrors` field.
    */
-  score: Record<string, unknown>;
+  score: { [key: string]: unknown };
 
   /**
    * Validation errors found in the Score data. If present, the Score doesn't fully
@@ -90,7 +90,7 @@ export interface ScoreUpsertParams {
    * conform to the output schema defined by the associated MetricConfig. If it
    * doesn't, validation errors will be captured in the `validationErrors` field.
    */
-  score: Record<string, unknown>;
+  score: { [key: string]: unknown };
 }
 
 export declare namespace Scores {

package/src/resources/shared.ts

@@ -6,7 +6,7 @@
 export interface APIError {
   code: string;
 
-  details: Record<string, unknown>;
+  details: { [key: string]: unknown };
 
   message: string;
 }

package/src/resources/systems/systems.ts

@@ -191,7 +191,7 @@ export interface System {
   /**
    * The schema of the system's configuration.
    */
-  configSchema: Record<string, unknown>;
+  configSchema: { [key: string]: unknown };
 
   /**
    * The description of the system.
@@ -201,7 +201,7 @@ export interface System {
   /**
    * The schema of the system's inputs.
    */
-  inputSchema: Record<string, unknown>;
+  inputSchema: { [key: string]: unknown };
 
   /**
    * The name of the system.
@@ -211,7 +211,7 @@ export interface System {
   /**
    * The schema of the system's outputs.
    */
-  outputSchema: Record<string, unknown>;
+  outputSchema: { [key: string]: unknown };
 }
 
 export interface SystemDeleteResponse {
@@ -225,7 +225,7 @@ export interface SystemCreateParams {
   /**
    * The schema of the system's configuration.
    */
-  configSchema: Record<string, unknown>;
+  configSchema: { [key: string]: unknown };
 
   /**
    * The description of the system.
@@ -235,7 +235,7 @@ export interface SystemCreateParams {
   /**
    * The schema of the system's inputs.
    */
-  inputSchema: Record<string, unknown>;
+  inputSchema: { [key: string]: unknown };
 
   /**
    * The name of the system.
@@ -245,14 +245,14 @@ export interface SystemCreateParams {
   /**
    * The schema of the system's outputs.
    */
-  outputSchema: Record<string, unknown>;
+  outputSchema: { [key: string]: unknown };
 }
 
 export interface SystemUpdateParams {
   /**
    * The schema of the system's configuration.
    */
-  configSchema?: Record<string, unknown>;
+  configSchema?: { [key: string]: unknown };
 
   /**
    * The description of the system.
@@ -262,7 +262,7 @@ export interface SystemUpdateParams {
   /**
    * The schema of the system's inputs.
    */
-  inputSchema?: Record<string, unknown>;
+  inputSchema?: { [key: string]: unknown };
 
   /**
    * The name of the system.
@@ -272,7 +272,7 @@ export interface SystemUpdateParams {
   /**
    * The schema of the system's outputs.
    */
-  outputSchema?: Record<string, unknown>;
+  outputSchema?: { [key: string]: unknown };
 }
 
 export interface SystemListParams extends PaginatedResponseParams {}

package/src/resources/systems/versions.ts

@@ -109,7 +109,7 @@ export interface SystemVersion {
   /**
    * The configuration of the system version.
    */
-  config: Record<string, unknown>;
+  config: { [key: string]: unknown };
 
   /**
    * The name of the system version.
@@ -146,7 +146,7 @@ export interface VersionCreateParams {
   /**
    * The configuration of the system version.
    */
-  config: Record<string, unknown>;
+  config: { [key: string]: unknown };
 
   /**
    * The name of the system version.

package/src/resources/testcases.ts

@@ -136,18 +136,18 @@ export interface Testcase {
    * Derived from data based on the Testset's fieldMapping. Contains all fields
    * marked as expected outputs, including those with validation errors.
    */
-  expected: Record<string, unknown>;
+  expected: { [key: string]: unknown };
 
   /**
    * Derived from data based on the Testset's fieldMapping. Contains all fields
    * marked as inputs, including those with validation errors.
    */
-  inputs: Record<string, unknown>;
+  inputs: { [key: string]: unknown };
 
   /**
    * The JSON data of the Testcase, which is validated against the Testset's schema.
    */
-  jsonData: Record<string, unknown>;
+  jsonData: { [key: string]: unknown };
 
   /**
    * The ID of the Testset this Testcase belongs to.
@@ -198,7 +198,7 @@ export namespace TestcaseCreateParams {
     /**
      * The JSON data of the Testcase, which is validated against the Testset's schema.
      */
-    jsonData: Record<string, unknown>;
+    jsonData: { [key: string]: unknown };
   }
 }
 
@@ -206,7 +206,7 @@ export interface TestcaseUpdateParams {
   /**
    * The JSON data of the Testcase, which is validated against the Testset's schema.
    */
-  jsonData: Record<string, unknown>;
+  jsonData: { [key: string]: unknown };
 }
 
 export interface TestcaseListParams extends PaginatedResponseParams {}

package/src/resources/testsets.ts

@@ -162,7 +162,7 @@ export interface Testset {
   /**
    * The JSON schema for each Testcase in the Testset.
    */
-  jsonSchema: Record<string, unknown>;
+  jsonSchema: { [key: string]: unknown };
 
   /**
    * The name of the Testset.
@@ -215,7 +215,7 @@ export interface TestsetCreateParams {
   /**
    * The JSON schema for each Testcase in the Testset.
    */
-  jsonSchema: Record<string, unknown>;
+  jsonSchema: { [key: string]: unknown };
 
   /**
    * The name of the Testset.
@@ -261,7 +261,7 @@ export interface TestsetUpdateParams {
   /**
    * The JSON schema for each Testcase in the Testset.
    */
-  jsonSchema?: Record<string, unknown>;
+  jsonSchema?: { [key: string]: unknown };
 
   /**
    * The name of the Testset.

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = '1.0.0'; // x-release-please-version
+export const VERSION = '1.2.0'; // x-release-please-version

package/version.d.mts

@@ -1,2 +1,2 @@
-export declare const VERSION = "1.0.0";
+export declare const VERSION = "1.2.0";
 //# sourceMappingURL=version.d.mts.map

package/version.d.ts

@@ -1,2 +1,2 @@
-export declare const VERSION = "1.0.0";
+export declare const VERSION = "1.2.0";
 //# sourceMappingURL=version.d.ts.map

package/version.js

@@ -1,5 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = '1.0.0'; // x-release-please-version
+exports.VERSION = '1.2.0'; // x-release-please-version
 //# sourceMappingURL=version.js.map

package/version.mjs

@@ -1,2 +1,2 @@
-export const VERSION = '1.0.0'; // x-release-please-version
+export const VERSION = '1.2.0'; // x-release-please-version
 //# sourceMappingURL=version.mjs.map