scorecard-ai 1.0.0-alpha.8 → 1.0.0

package/resources/metrics.d.mts

@@ -0,0 +1,421 @@
+import { APIResource } from "../core/resource.mjs";
+import { APIPromise } from "../core/api-promise.mjs";
+import { RequestOptions } from "../internal/request-options.mjs";
+export declare class Metrics extends APIResource {
+    /**
+     * Create a new Metric for evaluating system outputs.
+     *
+     * @example
+     * ```ts
+     * const metric = await client.metrics.create('314', {
+     *   evalType: 'ai',
+     *   name: 'Response Accuracy',
+     *   outputType: 'boolean',
+     *   promptTemplate:
+     *     'Please evaluate if the following response is factually accurate: {{outputs.response}}',
+     *   description:
+     *     'Evaluates if the response is factually accurate',
+     *   evalModelName: 'gpt-4o',
+     *   guidelines:
+     *     'Check if the response contains factually correct information',
+     *   temperature: 0.1,
+     * });
+     * ```
+     */
+    create(projectID: string, body: MetricCreateParams, options?: RequestOptions): APIPromise<Metric>;
+}
+/**
+ * A Metric defines how to evaluate system outputs against expected results.
+ */
+export type Metric = Metric.AIIntMetric | Metric.HumanIntMetric | Metric.HeuristicIntMetric | Metric.AIBooleanMetric | Metric.HumanBooleanMetric | Metric.HeuristicBooleanMetric;
+export declare namespace Metric {
+    /**
+     * A Metric with AI evaluation and integer output.
+     */
+    interface AIIntMetric {
+        /**
+         * The ID of the Metric.
+         */
+        id: string;
+        /**
+         * The description of the Metric.
+         */
+        description: string | null;
+        /**
+         * The AI model to use for evaluation.
+         */
+        evalModelName: string;
+        /**
+         * AI-based evaluation type.
+         */
+        evalType: 'ai';
+        /**
+         * Guidelines for AI evaluation on how to score the metric.
+         */
+        guidelines: string | null;
+        /**
+         * The name of the Metric.
+         */
+        name: string;
+        /**
+         * Integer output type.
+         */
+        outputType: 'int';
+        /**
+         * The threshold for determining pass/fail from integer scores (1-5).
+         */
+        passingThreshold: number;
+        /**
+         * The complete prompt template for AI evaluation. Should include placeholders for
+         * dynamic content.
+         */
+        promptTemplate: string;
+        /**
+         * The temperature for AI evaluation (0-2).
+         */
+        temperature: number;
+    }
+    /**
+     * A Metric with human evaluation and integer output.
+     */
+    interface HumanIntMetric {
+        /**
+         * The ID of the Metric.
+         */
+        id: string;
+        /**
+         * The description of the Metric.
+         */
+        description: string | null;
+        /**
+         * Human-based evaluation type.
+         */
+        evalType: 'human';
+        /**
+         * The name of the Metric.
+         */
+        name: string;
+        /**
+         * Integer output type.
+         */
+        outputType: 'int';
+        /**
+         * The threshold for determining pass/fail from integer scores (1-5).
+         */
+        passingThreshold: number;
+        /**
+         * Guidelines for human evaluators.
+         */
+        guidelines?: string;
+    }
+    /**
+     * A Metric with heuristic evaluation and integer output.
+     */
+    interface HeuristicIntMetric {
+        /**
+         * The ID of the Metric.
+         */
+        id: string;
+        /**
+         * The description of the Metric.
+         */
+        description: string | null;
+        /**
+         * Heuristic-based evaluation type.
+         */
+        evalType: 'heuristic';
+        /**
+         * The name of the Metric.
+         */
+        name: string;
+        /**
+         * Integer output type.
+         */
+        outputType: 'int';
+        /**
+         * The threshold for determining pass/fail from integer scores (1-5).
+         */
+        passingThreshold: number;
+        /**
+         * Optional guidelines for heuristic evaluation logic.
+         */
+        guidelines?: string;
+    }
+    /**
+     * A Metric with AI evaluation and boolean output.
+     */
+    interface AIBooleanMetric {
+        /**
+         * The ID of the Metric.
+         */
+        id: string;
+        /**
+         * The description of the Metric.
+         */
+        description: string | null;
+        /**
+         * The AI model to use for evaluation.
+         */
+        evalModelName: string;
+        /**
+         * AI-based evaluation type.
+         */
+        evalType: 'ai';
+        /**
+         * Guidelines for AI evaluation on how to score the metric.
+         */
+        guidelines: string | null;
+        /**
+         * The name of the Metric.
+         */
+        name: string;
+        /**
+         * Boolean output type.
+         */
+        outputType: 'boolean';
+        /**
+         * The complete prompt template for AI evaluation. Should include placeholders for
+         * dynamic content.
+         */
+        promptTemplate: string;
+        /**
+         * The temperature for AI evaluation (0-2).
+         */
+        temperature: number;
+    }
+    /**
+     * A Metric with human evaluation and boolean output.
+     */
+    interface HumanBooleanMetric {
+        /**
+         * The ID of the Metric.
+         */
+        id: string;
+        /**
+         * The description of the Metric.
+         */
+        description: string | null;
+        /**
+         * Human-based evaluation type.
+         */
+        evalType: 'human';
+        /**
+         * The name of the Metric.
+         */
+        name: string;
+        /**
+         * Boolean output type.
+         */
+        outputType: 'boolean';
+        /**
+         * Guidelines for human evaluators.
+         */
+        guidelines?: string;
+    }
+    /**
+     * A Metric with heuristic evaluation and boolean output.
+     */
+    interface HeuristicBooleanMetric {
+        /**
+         * The ID of the Metric.
+         */
+        id: string;
+        /**
+         * The description of the Metric.
+         */
+        description: string | null;
+        /**
+         * Heuristic-based evaluation type.
+         */
+        evalType: 'heuristic';
+        /**
+         * The name of the Metric.
+         */
+        name: string;
+        /**
+         * Boolean output type.
+         */
+        outputType: 'boolean';
+        /**
+         * Optional guidelines for heuristic evaluation logic.
+         */
+        guidelines?: string;
+    }
+}
+export type MetricCreateParams = MetricCreateParams.AIIntMetric | MetricCreateParams.HumanIntMetric | MetricCreateParams.HeuristicIntMetric | MetricCreateParams.AIBooleanMetric | MetricCreateParams.HumanBooleanMetric | MetricCreateParams.HeuristicBooleanMetric;
+export declare namespace MetricCreateParams {
+    interface AIIntMetric {
+        /**
+         * AI-based evaluation type.
+         */
+        evalType: 'ai';
+        /**
+         * The name of the Metric.
+         */
+        name: string;
+        /**
+         * Integer output type.
+         */
+        outputType: 'int';
+        /**
+         * The complete prompt template for AI evaluation. Should include placeholders for
+         * dynamic content.
+         */
+        promptTemplate: string;
+        /**
+         * The description of the Metric.
+         */
+        description?: string | null;
+        /**
+         * The AI model to use for evaluation.
+         */
+        evalModelName?: string;
+        /**
+         * Guidelines for AI evaluation on how to score the metric.
+         */
+        guidelines?: string | null;
+        /**
+         * The threshold for determining pass/fail from integer scores (1-5).
+         */
+        passingThreshold?: number;
+        /**
+         * The temperature for AI evaluation (0-2).
+         */
+        temperature?: number;
+    }
+    interface HumanIntMetric {
+        /**
+         * Human-based evaluation type.
+         */
+        evalType: 'human';
+        /**
+         * The name of the Metric.
+         */
+        name: string;
+        /**
+         * Integer output type.
+         */
+        outputType: 'int';
+        /**
+         * The description of the Metric.
+         */
+        description?: string | null;
+        /**
+         * Guidelines for human evaluators.
+         */
+        guidelines?: string;
+        /**
+         * The threshold for determining pass/fail from integer scores (1-5).
+         */
+        passingThreshold?: number;
+    }
+    interface HeuristicIntMetric {
+        /**
+         * Heuristic-based evaluation type.
+         */
+        evalType: 'heuristic';
+        /**
+         * The name of the Metric.
+         */
+        name: string;
+        /**
+         * Integer output type.
+         */
+        outputType: 'int';
+        /**
+         * The description of the Metric.
+         */
+        description?: string | null;
+        /**
+         * Optional guidelines for heuristic evaluation logic.
+         */
+        guidelines?: string;
+        /**
+         * The threshold for determining pass/fail from integer scores (1-5).
+         */
+        passingThreshold?: number;
+    }
+    interface AIBooleanMetric {
+        /**
+         * AI-based evaluation type.
+         */
+        evalType: 'ai';
+        /**
+         * The name of the Metric.
+         */
+        name: string;
+        /**
+         * Boolean output type.
+         */
+        outputType: 'boolean';
+        /**
+         * The complete prompt template for AI evaluation. Should include placeholders for
+         * dynamic content.
+         */
+        promptTemplate: string;
+        /**
+         * The description of the Metric.
+         */
+        description?: string | null;
+        /**
+         * The AI model to use for evaluation.
+         */
+        evalModelName?: string;
+        /**
+         * Guidelines for AI evaluation on how to score the metric.
+         */
+        guidelines?: string | null;
+        /**
+         * The temperature for AI evaluation (0-2).
+         */
+        temperature?: number;
+    }
+    interface HumanBooleanMetric {
+        /**
+         * Human-based evaluation type.
+         */
+        evalType: 'human';
+        /**
+         * The name of the Metric.
+         */
+        name: string;
+        /**
+         * Boolean output type.
+         */
+        outputType: 'boolean';
+        /**
+         * The description of the Metric.
+         */
+        description?: string | null;
+        /**
+         * Guidelines for human evaluators.
+         */
+        guidelines?: string;
+    }
+    interface HeuristicBooleanMetric {
+        /**
+         * Heuristic-based evaluation type.
+         */
+        evalType: 'heuristic';
+        /**
+         * The name of the Metric.
+         */
+        name: string;
+        /**
+         * Boolean output type.
+         */
+        outputType: 'boolean';
+        /**
+         * The description of the Metric.
+         */
+        description?: string | null;
+        /**
+         * Optional guidelines for heuristic evaluation logic.
+         */
+        guidelines?: string;
+    }
+}
+export declare namespace Metrics {
+    export { type Metric as Metric, type MetricCreateParams as MetricCreateParams };
+}
+//# sourceMappingURL=metrics.d.mts.map

package/resources/metrics.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.mts","sourceRoot":"","sources":["../src/resources/metrics.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,cAAc,EAAE;AAGzB,qBAAa,OAAQ,SAAQ,WAAW;IACtC;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;CAGlG;AAED;;GAEG;AACH,MAAM,MAAM,MAAM,GACd,MAAM,CAAC,WAAW,GAClB,MAAM,CAAC,cAAc,GACrB,MAAM,CAAC,kBAAkB,GACzB,MAAM,CAAC,eAAe,GACtB,MAAM,CAAC,kBAAkB,GACzB,MAAM,CAAC,sBAAsB,CAAC;AAElC,yBAAiB,MAAM,CAAC;IACtB;;OAEG;IACH,UAAiB,WAAW;QAC1B;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,aAAa,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,QAAQ,EAAE,IAAI,CAAC;QAEf;;WAEG;QACH,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAE1B;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,gBAAgB,EAAE,MAAM,CAAC;QAEzB;;;WAGG;QACH,cAAc,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,WAAW,EAAE,MAAM,CAAC;KACrB;IAED;;OAEG;IACH,UAAiB,cAAc;QAC7B;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,gBAAgB,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;IAED;;OAEG;IACH,UAAiB,kBAAkB;QACjC;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,QAAQ,EAAE,WAAW,CAAC;QAEtB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,gBAAgB,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;IAED;;OAEG;IACH,UAAiB,eAAe;QAC9B;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,aAAa,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,QAAQ,EAAE,IAAI,CAAC;QAEf;;WAEG;QACH,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAE1B;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;;WAGG;QACH,cAAc,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,WAAW,EAAE,MAAM,CAAC;KACrB;IAED;;OAEG;IACH,UAAiB,kBAAkB;QACjC;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;IAED;;OAEG;IACH,UAAiB,sBAAsB;QACrC;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,QAAQ,EAAE,WAAW,CAAC;QAEtB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;CACF;AAED,MAAM,MAAM,kBAAkB,GAC1B,kBAAkB,CAAC,WAAW,GAC9B,kBAAkB,CAAC,cAAc,GACjC,kBAAkB,CAAC,kBAAkB,GACrC,kBAAkB,CAAC,eAAe,GAClC,kBAAkB,CAAC,kBAAkB,GACrC,kBAAkB,CAAC,sBAAsB,CAAC;AAE9C,MAAM,CAAC,OAAO,WAAW,kBAAkB,CAAC;IAC1C,UAAiB,WAAW;QAC1B;;WAEG;QACH,QAAQ,EAAE,IAAI,CAAC;QAEf;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;;WAGG;QACH,cAAc,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,aAAa,CAAC,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB;IAED,UAAiB,cAAc;QAC7B;;WAEG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B;IAED,UAAiB,kBAAkB;QACjC;;WAEG;QACH,QAAQ,EAAE,WAAW,CAAC;QAEtB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B;IAED,UAAiB,eAAe;QAC9B;;WAEG;QACH,QAAQ,EAAE,IAAI,CAAC;QAEf;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;;WAGG;QACH,cAAc,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,aAAa,CAAC,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB;IAED,UAAiB,kBAAkB;QACjC;;WAEG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;IAED,UAAiB,sBAAsB;QACrC;;WAEG;QACH,QAAQ,EAAE,WAAW,CAAC;QAEtB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;CACF;AAED,MAAM,CAAC,OAAO,WAAW,OAAO,CAAC;IAC/B,OAAO,EAAE,KAAK,MAAM,IAAI,MAAM,EAAE,KAAK,kBAAkB,IAAI,kBAAkB,EAAE,CAAC;CACjF"}