scorecard-ai 1.0.0 → 1.1.0

package/src/resources/metrics.ts

@@ -7,7 +7,8 @@ import { path } from '../internal/utils/path';
 
 export class Metrics extends APIResource {
   /**
-   * Create a new Metric for evaluating system outputs.
+   * Create a new Metric for evaluating system outputs. The structure of a metric
+   * depends on the evalType and outputType of the metric.
    *
    * @example
    * ```ts
@@ -29,6 +30,25 @@ export class Metrics extends APIResource {
   create(projectID: string, body: MetricCreateParams, options?: RequestOptions): APIPromise<Metric> {
     return this._client.post(path`/projects/${projectID}/metrics`, { body, ...options });
   }
+
+  /**
+   * Update an existing Metric. You must specify the evalType and outputType of the
+   * metric. The structure of a metric depends on the evalType and outputType of the
+   * metric.
+   *
+   * @example
+   * ```ts
+   * const metric = await client.metrics.update('321', {
+   *   evalType: 'ai',
+   *   outputType: 'boolean',
+   *   promptTemplate:
+   *     'Using the following guidelines, evaluate the response: {{ guidelines }}\n\nResponse: {{ outputs.response }}\n\nIdeal answer: {{ expected.idealResponse }}',
+   * });
+   * ```
+   */
+  update(metricID: string, body: MetricUpdateParams, options?: RequestOptions): APIPromise<Metric> {
+    return this._client.patch(path`/metrics/${metricID}`, { body, ...options });
+  }
 }
 
 /**
@@ -520,6 +540,229 @@ export declare namespace MetricCreateParams {
   }
 }
 
+export type MetricUpdateParams =
+  | MetricUpdateParams.AIIntMetric
+  | MetricUpdateParams.HumanIntMetric
+  | MetricUpdateParams.HeuristicIntMetric
+  | MetricUpdateParams.AIBooleanMetric
+  | MetricUpdateParams.HumanBooleanMetric
+  | MetricUpdateParams.HeuristicBooleanMetric;
+
+export declare namespace MetricUpdateParams {
+  export interface AIIntMetric {
+    /**
+     * AI-based evaluation type.
+     */
+    evalType: 'ai';
+
+    /**
+     * Integer output type.
+     */
+    outputType: 'int';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * The AI model to use for evaluation.
+     */
+    evalModelName?: string;
+
+    /**
+     * Guidelines for AI evaluation on how to score the metric.
+     */
+    guidelines?: string | null;
+
+    /**
+     * The name of the Metric.
+     */
+    name?: string;
+
+    /**
+     * The threshold for determining pass/fail from integer scores (1-5).
+     */
+    passingThreshold?: number;
+
+    /**
+     * The complete prompt template for AI evaluation. Should include placeholders for
+     * dynamic content.
+     */
+    promptTemplate?: string;
+
+    /**
+     * The temperature for AI evaluation (0-2).
+     */
+    temperature?: number;
+  }
+
+  export interface HumanIntMetric {
+    /**
+     * Human-based evaluation type.
+     */
+    evalType: 'human';
+
+    /**
+     * Integer output type.
+     */
+    outputType: 'int';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * Guidelines for human evaluators.
+     */
+    guidelines?: string;
+
+    /**
+     * The name of the Metric.
+     */
+    name?: string;
+
+    /**
+     * The threshold for determining pass/fail from integer scores (1-5).
+     */
+    passingThreshold?: number;
+  }
+
+  export interface HeuristicIntMetric {
+    /**
+     * Heuristic-based evaluation type.
+     */
+    evalType: 'heuristic';
+
+    /**
+     * Integer output type.
+     */
+    outputType: 'int';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * Optional guidelines for heuristic evaluation logic.
+     */
+    guidelines?: string;
+
+    /**
+     * The name of the Metric.
+     */
+    name?: string;
+
+    /**
+     * The threshold for determining pass/fail from integer scores (1-5).
+     */
+    passingThreshold?: number;
+  }
+
+  export interface AIBooleanMetric {
+    /**
+     * AI-based evaluation type.
+     */
+    evalType: 'ai';
+
+    /**
+     * Boolean output type.
+     */
+    outputType: 'boolean';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * The AI model to use for evaluation.
+     */
+    evalModelName?: string;
+
+    /**
+     * Guidelines for AI evaluation on how to score the metric.
+     */
+    guidelines?: string | null;
+
+    /**
+     * The name of the Metric.
+     */
+    name?: string;
+
+    /**
+     * The complete prompt template for AI evaluation. Should include placeholders for
+     * dynamic content.
+     */
+    promptTemplate?: string;
+
+    /**
+     * The temperature for AI evaluation (0-2).
+     */
+    temperature?: number;
+  }
+
+  export interface HumanBooleanMetric {
+    /**
+     * Human-based evaluation type.
+     */
+    evalType: 'human';
+
+    /**
+     * Boolean output type.
+     */
+    outputType: 'boolean';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * Guidelines for human evaluators.
+     */
+    guidelines?: string;
+
+    /**
+     * The name of the Metric.
+     */
+    name?: string;
+  }
+
+  export interface HeuristicBooleanMetric {
+    /**
+     * Heuristic-based evaluation type.
+     */
+    evalType: 'heuristic';
+
+    /**
+     * Boolean output type.
+     */
+    outputType: 'boolean';
+
+    /**
+     * The description of the Metric.
+     */
+    description?: string | null;
+
+    /**
+     * Optional guidelines for heuristic evaluation logic.
+     */
+    guidelines?: string;
+
+    /**
+     * The name of the Metric.
+     */
+    name?: string;
+  }
+}
+
 export declare namespace Metrics {
-  export { type Metric as Metric, type MetricCreateParams as MetricCreateParams };
+  export {
+    type Metric as Metric,
+    type MetricCreateParams as MetricCreateParams,
+    type MetricUpdateParams as MetricUpdateParams,
+  };
 }

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = '1.0.0'; // x-release-please-version
+export const VERSION = '1.1.0'; // x-release-please-version

package/version.d.mts

@@ -1,2 +1,2 @@
-export declare const VERSION = "1.0.0";
+export declare const VERSION = "1.1.0";
 //# sourceMappingURL=version.d.mts.map

package/version.d.ts

@@ -1,2 +1,2 @@
-export declare const VERSION = "1.0.0";
+export declare const VERSION = "1.1.0";
 //# sourceMappingURL=version.d.ts.map

package/version.js

@@ -1,5 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = '1.0.0'; // x-release-please-version
+exports.VERSION = '1.1.0'; // x-release-please-version
 //# sourceMappingURL=version.js.map

package/version.mjs

@@ -1,2 +1,2 @@
-export const VERSION = '1.0.0'; // x-release-please-version
+export const VERSION = '1.1.0'; // x-release-please-version
 //# sourceMappingURL=version.mjs.map