telnyx 7.28.0 → 7.29.0

package/resources/ai/tests/test-framework/v1/metrics.d.ts

@@ -0,0 +1,339 @@
+import { APIResource } from "../../../../../core/resource.js";
+import { APIPromise } from "../../../../../core/api-promise.js";
+import { RequestOptions } from "../../../../../internal/request-options.js";
+/**
+ * Test agents, evaluator scenarios, metrics, and run results for the AI test framework.
+ */
+export declare class Metrics extends APIResource {
+    /**
+     * Returns the metrics available in your organization, optionally filtered by test
+     * agent. Each metric scores one aspect of a conversation or scenario run.
+     *
+     * @example
+     * ```ts
+     * const aiTestMetrics =
+     *   await client.ai.tests.testFramework.v1.metrics.list();
+     * ```
+     */
+    list(query?: MetricListParams | null | undefined, options?: RequestOptions): APIPromise<MetricListResponse>;
+    /**
+     * Creates a metric in your organization.
+     *
+     * @example
+     * ```ts
+     * const aiTestMetric =
+     *   await client.ai.tests.testFramework.v1.metrics.create({
+     *     name: 'Escalates on refund denial',
+     *     agents: [1259],
+     *     description:
+     *       "Returns a score for whether the assistant escalates when the caller's refund is denied.",
+     *     eval_type: 'continuous_qualitative',
+     *     observability_enabled: true,
+     *     prompt:
+     *       "Score the assistant's escalation behaviour between 0.0 and 1.0.",
+     *     simulation_enabled: true,
+     *     type: 'llm_judge',
+     *   });
+     * ```
+     */
+    create(body: MetricCreateParams, options?: RequestOptions): APIPromise<AITestMetric>;
+    /**
+     * Creates several metrics in one request. Accepts an array of metric objects and
+     * returns the created metrics.
+     *
+     * @example
+     * ```ts
+     * const aiTestMetrics =
+     *   await client.ai.tests.testFramework.v1.metrics.bulkCreate(
+     *     { body: [{ name: 'Greets within 5 seconds' }] },
+     *   );
+     * ```
+     */
+    bulkCreate(params: MetricBulkCreateParams, options?: RequestOptions): APIPromise<MetricBulkCreateResponse>;
+    /**
+     * Starts background generation of suggested metrics for a test agent. Returns a
+     * progress ID to poll with the get-test-metric-generation-progress endpoint.
+     *
+     * @example
+     * ```ts
+     * const scenarioGenerationResponse =
+     *   await client.ai.tests.testFramework.v1.metrics.generateMetrics(
+     *     {
+     *       agent_id: 1259,
+     *       num_metrics: 5,
+     *       scenarios: [9512],
+     *     },
+     *   );
+     * ```
+     */
+    generateMetrics(body: MetricGenerateMetricsParams, options?: RequestOptions): APIPromise<ScenarioGenerationResponse>;
+    /**
+     * Returns the status of a background metric-generation job started with the
+     * generate-test-metrics endpoint.
+     *
+     * @example
+     * ```ts
+     * const response =
+     *   await client.ai.tests.testFramework.v1.metrics.retrieveGenerateMetricsProgress(
+     *     { progress_id: 'progress_id' },
+     *   );
+     * ```
+     */
+    retrieveGenerateMetricsProgress(query: MetricRetrieveGenerateMetricsProgressParams, options?: RequestOptions): APIPromise<MetricRetrieveGenerateMetricsProgressResponse>;
+    /**
+     * Deletes the specified metric.
+     *
+     * @example
+     * ```ts
+     * const metric =
+     *   await client.ai.tests.testFramework.v1.metrics.delete(0);
+     * ```
+     */
+    delete(id: number, options?: RequestOptions): APIPromise<unknown>;
+    /**
+     * Returns the specified metric.
+     *
+     * @example
+     * ```ts
+     * const aiTestMetric =
+     *   await client.ai.tests.testFramework.v1.metrics.retrieve(
+     *     0,
+     *   );
+     * ```
+     */
+    retrieve(id: number, options?: RequestOptions): APIPromise<AITestMetric>;
+    /**
+     * Updates the specified metric.
+     *
+     * @example
+     * ```ts
+     * const aiTestMetric =
+     *   await client.ai.tests.testFramework.v1.metrics.update(0, {
+     *     audio_enabled: true,
+     *     name: 'Escalates on refund denial',
+     *     prompt:
+     *       "Score the assistant's escalation behaviour between 0.0 and 1.0, considering tone.",
+     *   });
+     * ```
+     */
+    update(id: number, body: MetricUpdateParams, options?: RequestOptions): APIPromise<AITestMetric>;
+}
+export interface AITestMetric {
+    id: number;
+    /**
+     * What the metric scores: numeric, continuous_qualitative, or
+     * binary_workflow_adherence.
+     */
+    eval_type: string;
+    name: string;
+    /**
+     * How the metric is evaluated: custom_code or llm_judge.
+     */
+    type: string;
+    /**
+     * Whether the metric is attached to newly created test agents by default.
+     */
+    add_to_new_agents?: boolean | null;
+    agent?: number | null;
+    agent_names?: Array<string>;
+    /**
+     * Test agent IDs the metric is attached to.
+     */
+    agents?: Array<number>;
+    alert_filters?: {
+        [key: string]: unknown;
+    } | null;
+    alert_type?: string;
+    audio_enabled?: boolean;
+    configuration?: {
+        [key: string]: unknown;
+    } | null;
+    description?: string;
+    display_order?: number;
+    /**
+     * Allowed values for qualitative enum metrics.
+     */
+    enum_values?: Array<string>;
+    evaluation_trigger?: string;
+    evaluation_trigger_custom_code?: string;
+    evaluation_trigger_prompt?: string;
+    /**
+     * Smoothing factor for the alert trend.
+     */
+    ewma_alpha?: number;
+    /**
+     * Built-in evaluation function used by custom_code metrics.
+     */
+    function_name?: string | null;
+    kb_file_ids?: Array<number>;
+    knowledge_base_files?: Array<string>;
+    metric_cost?: number;
+    observability_enabled?: boolean;
+    overall_score?: number | string;
+    priority_assignment_prompt?: string;
+    project?: number | null;
+    /**
+     * Judge prompt used by llm_judge metrics.
+     */
+    prompt?: string;
+    sample_rate?: number | null;
+    sampling_enabled?: boolean;
+    significant_change_alert_direction?: string;
+    significant_change_alert_status?: string;
+    simulation_enabled?: boolean;
+    /**
+     * Stable identifier of a built-in metric.
+     */
+    slug?: string | null;
+    /**
+     * Standard-deviation multiplier for significant-change alerts.
+     */
+    std_multiplier?: number;
+    total_score?: number | string;
+    trigger_type?: string;
+    user_defined?: boolean;
+    uses_llm_judge?: boolean;
+    /**
+     * Sliding window size the alert status is computed over.
+     */
+    window_size?: number;
+}
+export interface CreateAITestMetricRequest {
+    name: string;
+    add_to_new_agents?: boolean | null;
+    agents?: Array<number>;
+    assistant_id?: string;
+    audio_enabled?: boolean;
+    description?: string;
+    display_order?: number;
+    enum_values?: Array<string>;
+    /**
+     * What the metric scores: numeric, continuous_qualitative, or
+     * binary_workflow_adherence.
+     */
+    eval_type?: string;
+    observability_enabled?: boolean;
+    /**
+     * Judge prompt used by llm_judge metrics.
+     */
+    prompt?: string;
+    sampling_enabled?: boolean;
+    simulation_enabled?: boolean;
+    /**
+     * How the metric is evaluated: custom_code or llm_judge.
+     */
+    type?: string;
+}
+export interface ScenarioGenerationResponse {
+    /**
+     * Identifier to poll generation progress with.
+     */
+    progress_id: string;
+}
+export type MetricListResponse = Array<AITestMetric>;
+export type MetricDeleteResponse = unknown;
+export type MetricBulkCreateResponse = Array<AITestMetric>;
+export interface MetricRetrieveGenerateMetricsProgressResponse {
+    agent_id: number;
+    current_step: string;
+    error: string | null;
+    metrics: Array<MetricRetrieveGenerateMetricsProgressResponse.Metric>;
+    status: 'queued' | 'processing' | 'completed' | 'failed';
+}
+export declare namespace MetricRetrieveGenerateMetricsProgressResponse {
+    interface Metric {
+        agent?: number;
+        description?: string;
+        eval_type?: string;
+        name?: string;
+        scenarios?: Array<number>;
+    }
+}
+export interface MetricListParams {
+    /**
+     * Filter metrics attached to the specified test agent.
+     */
+    agent_id?: number;
+    /**
+     * Filter metrics attached to the specified Telnyx assistant.
+     */
+    assistant_id?: string;
+}
+export interface MetricCreateParams {
+    name: string;
+    add_to_new_agents?: boolean | null;
+    agents?: Array<number>;
+    assistant_id?: string;
+    audio_enabled?: boolean;
+    description?: string;
+    display_order?: number;
+    enum_values?: Array<string>;
+    /**
+     * What the metric scores: numeric, continuous_qualitative, or
+     * binary_workflow_adherence.
+     */
+    eval_type?: string;
+    observability_enabled?: boolean;
+    /**
+     * Judge prompt used by llm_judge metrics.
+     */
+    prompt?: string;
+    sampling_enabled?: boolean;
+    simulation_enabled?: boolean;
+    /**
+     * How the metric is evaluated: custom_code or llm_judge.
+     */
+    type?: string;
+}
+export interface MetricBulkCreateParams {
+    body: Array<CreateAITestMetricRequest>;
+}
+export interface MetricGenerateMetricsParams {
+    /**
+     * Test agent to suggest metrics for.
+     */
+    agent_id: number;
+    /**
+     * How many metrics to suggest.
+     */
+    num_metrics: number;
+    /**
+     * Scenario IDs to base the suggestions on.
+     */
+    scenarios?: Array<number>;
+}
+export interface MetricRetrieveGenerateMetricsProgressParams {
+    /**
+     * Progress ID returned by the generate-test-metrics endpoint.
+     */
+    progress_id: string;
+}
+export interface MetricUpdateParams {
+    add_to_new_agents?: boolean | null;
+    agents?: Array<number>;
+    audio_enabled?: boolean;
+    description?: string;
+    display_order?: number;
+    enum_values?: Array<string>;
+    /**
+     * What the metric scores: numeric, continuous_qualitative, or
+     * binary_workflow_adherence.
+     */
+    eval_type?: string;
+    name?: string;
+    observability_enabled?: boolean;
+    /**
+     * Judge prompt used by llm_judge metrics.
+     */
+    prompt?: string;
+    sampling_enabled?: boolean;
+    simulation_enabled?: boolean;
+    /**
+     * How the metric is evaluated: custom_code or llm_judge.
+     */
+    type?: string;
+}
+export declare namespace Metrics {
+    export { type AITestMetric as AITestMetric, type CreateAITestMetricRequest as CreateAITestMetricRequest, type ScenarioGenerationResponse as ScenarioGenerationResponse, type MetricListResponse as MetricListResponse, type MetricDeleteResponse as MetricDeleteResponse, type MetricBulkCreateResponse as MetricBulkCreateResponse, type MetricRetrieveGenerateMetricsProgressResponse as MetricRetrieveGenerateMetricsProgressResponse, type MetricListParams as MetricListParams, type MetricCreateParams as MetricCreateParams, type MetricBulkCreateParams as MetricBulkCreateParams, type MetricGenerateMetricsParams as MetricGenerateMetricsParams, type MetricRetrieveGenerateMetricsProgressParams as MetricRetrieveGenerateMetricsProgressParams, type MetricUpdateParams as MetricUpdateParams, };
+}
+//# sourceMappingURL=metrics.d.ts.map

package/resources/ai/tests/test-framework/v1/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../../../src/resources/ai/tests/test-framework/v1/metrics.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,wCAAqC;AAC3D,OAAO,EAAE,UAAU,EAAE,2CAAwC;AAC7D,OAAO,EAAE,cAAc,EAAE,mDAAgD;AAGzE;;GAEG;AACH,qBAAa,OAAQ,SAAQ,WAAW;IACtC;;;;;;;;;OASG;IACH,IAAI,CACF,KAAK,GAAE,gBAAgB,GAAG,IAAI,GAAG,SAAc,EAC/C,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,kBAAkB,CAAC;IAIjC;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,IAAI,EAAE,kBAAkB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,YAAY,CAAC;IAIpF;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,MAAM,EAAE,sBAAsB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,wBAAwB,CAAC;IAK1G;;;;;;;;;;;;;;;OAeG;IACH,eAAe,CACb,IAAI,EAAE,2BAA2B,EACjC,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,0BAA0B,CAAC;IAIzC;;;;;;;;;;;OAWG;IACH,+BAA+B,CAC7B,KAAK,EAAE,2CAA2C,EAClD,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,6CAA6C,CAAC;IAO5D;;;;;;;;OAQG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC;IAIjE;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,YAAY,CAAC;IAIxE;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,YAAY,CAAC;CAGjG;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IAEX;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,iBAAiB,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAEnC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtB,WAAW,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAE5B;;OAEG;IACH,MAAM,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEvB,aAAa,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;IAElD,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,aAAa,CAAC,EAAE,OAAO,CAAC;IAExB,aAAa,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;IAElD,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;OAEG;IACH,WAAW,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAE5B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B,8BAA8B,CAAC,EAAE,MAAM,CAAC;IAExC,yBAAyB,CAAC,EAAE,MAAM,CAAC;IAEnC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE9B,WAAW,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAE5B,oBAAoB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAErC,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC,aAAa,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAEhC,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAExB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE5B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAE3B,kCAAkC,CAAC,EAAE,MAAM,CAAC;IAE5C,+BAA+B,CAAC,EAAE,MAAM,CAAC;IAEzC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAE7B;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAErB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAE9B,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,yBAAyB;IACxC,IAAI,EAAE,MAAM,CAAC;IAEb,iBAAiB,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAEnC,MAAM,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEvB,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,aAAa,CAAC,EAAE,OAAO,CAAC;IAExB,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,WAAW,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAE5B;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAE3B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAE7B;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,0BAA0B;IACzC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,kBAAkB,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;AAErD,MAAM,MAAM,oBAAoB,GAAG,OAAO,CAAC;AAE3C,MAAM,MAAM,wBAAwB,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;AAE3D,MAAM,WAAW,6CAA6C;IAC5D,QAAQ,EAAE,MAAM,CAAC;IAEjB,YAAY,EAAE,MAAM,CAAC;IAErB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAErB,OAAO,EAAE,KAAK,CAAC,6CAA6C,CAAC,MAAM,CAAC,CAAC;IAErE,MAAM,EAAE,QAAQ,GAAG,YAAY,GAAG,WAAW,GAAG,QAAQ,CAAC;CAC1D;AAED,yBAAiB,6CAA6C,CAAC;IAC7D,UAAiB,MAAM;QACrB,KAAK,CAAC,EAAE,MAAM,CAAC;QAEf,WAAW,CAAC,EAAE,MAAM,CAAC;QAErB,SAAS,CAAC,EAAE,MAAM,CAAC;QAEnB,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd,SAAS,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;KAC3B;CACF;AAED,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IAEb,iBAAiB,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAEnC,MAAM,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEvB,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,aAAa,CAAC,EAAE,OAAO,CAAC;IAExB,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,WAAW,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAE5B;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAE3B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAE7B;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,KAAK,CAAC,yBAAyB,CAAC,CAAC;CACxC;AAED,MAAM,WAAW,2BAA2B;IAC1C;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,SAAS,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CAC3B;AAED,MAAM,WAAW,2CAA2C;IAC1D;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IACjC,iBAAiB,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAEnC,MAAM,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEvB,aAAa,CAAC,EAAE,OAAO,CAAC;IAExB,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,WAAW,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAE5B;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAE3B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAE7B;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,CAAC,OAAO,WAAW,OAAO,CAAC;IAC/B,OAAO,EACL,KAAK,YAAY,IAAI,YAAY,EACjC,KAAK,yBAAyB,IAAI,yBAAyB,EAC3D,KAAK,0BAA0B,IAAI,0BAA0B,EAC7D,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,oBAAoB,IAAI,oBAAoB,EACjD,KAAK,wBAAwB,IAAI,wBAAwB,EACzD,KAAK,6CAA6C,IAAI,6CAA6C,EACnG,KAAK,gBAAgB,IAAI,gBAAgB,EACzC,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,sBAAsB,IAAI,sBAAsB,EACrD,KAAK,2BAA2B,IAAI,2BAA2B,EAC/D,KAAK,2CAA2C,IAAI,2CAA2C,EAC/F,KAAK,kBAAkB,IAAI,kBAAkB,GAC9C,CAAC;CACH"}

package/resources/ai/tests/test-framework/v1/metrics.js

@@ -0,0 +1,145 @@
+"use strict";
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Metrics = void 0;
+const resource_1 = require("../../../../../core/resource.js");
+const path_1 = require("../../../../../internal/utils/path.js");
+/**
+ * Test agents, evaluator scenarios, metrics, and run results for the AI test framework.
+ */
+class Metrics extends resource_1.APIResource {
+    /**
+     * Returns the metrics available in your organization, optionally filtered by test
+     * agent. Each metric scores one aspect of a conversation or scenario run.
+     *
+     * @example
+     * ```ts
+     * const aiTestMetrics =
+     *   await client.ai.tests.testFramework.v1.metrics.list();
+     * ```
+     */
+    list(query = {}, options) {
+        return this._client.get('/ai/tests/test_framework/v1/metrics', { query, ...options });
+    }
+    /**
+     * Creates a metric in your organization.
+     *
+     * @example
+     * ```ts
+     * const aiTestMetric =
+     *   await client.ai.tests.testFramework.v1.metrics.create({
+     *     name: 'Escalates on refund denial',
+     *     agents: [1259],
+     *     description:
+     *       "Returns a score for whether the assistant escalates when the caller's refund is denied.",
+     *     eval_type: 'continuous_qualitative',
+     *     observability_enabled: true,
+     *     prompt:
+     *       "Score the assistant's escalation behaviour between 0.0 and 1.0.",
+     *     simulation_enabled: true,
+     *     type: 'llm_judge',
+     *   });
+     * ```
+     */
+    create(body, options) {
+        return this._client.post('/ai/tests/test_framework/v1/metrics', { body, ...options });
+    }
+    /**
+     * Creates several metrics in one request. Accepts an array of metric objects and
+     * returns the created metrics.
+     *
+     * @example
+     * ```ts
+     * const aiTestMetrics =
+     *   await client.ai.tests.testFramework.v1.metrics.bulkCreate(
+     *     { body: [{ name: 'Greets within 5 seconds' }] },
+     *   );
+     * ```
+     */
+    bulkCreate(params, options) {
+        const { body } = params;
+        return this._client.post('/ai/tests/test_framework/v1/metrics/bulk_create', { body: body, ...options });
+    }
+    /**
+     * Starts background generation of suggested metrics for a test agent. Returns a
+     * progress ID to poll with the get-test-metric-generation-progress endpoint.
+     *
+     * @example
+     * ```ts
+     * const scenarioGenerationResponse =
+     *   await client.ai.tests.testFramework.v1.metrics.generateMetrics(
+     *     {
+     *       agent_id: 1259,
+     *       num_metrics: 5,
+     *       scenarios: [9512],
+     *     },
+     *   );
+     * ```
+     */
+    generateMetrics(body, options) {
+        return this._client.post('/ai/tests/test_framework/v1/metrics/generate_metrics', { body, ...options });
+    }
+    /**
+     * Returns the status of a background metric-generation job started with the
+     * generate-test-metrics endpoint.
+     *
+     * @example
+     * ```ts
+     * const response =
+     *   await client.ai.tests.testFramework.v1.metrics.retrieveGenerateMetricsProgress(
+     *     { progress_id: 'progress_id' },
+     *   );
+     * ```
+     */
+    retrieveGenerateMetricsProgress(query, options) {
+        return this._client.get('/ai/tests/test_framework/v1/metrics/generate_metrics_progress', {
+            query,
+            ...options,
+        });
+    }
+    /**
+     * Deletes the specified metric.
+     *
+     * @example
+     * ```ts
+     * const metric =
+     *   await client.ai.tests.testFramework.v1.metrics.delete(0);
+     * ```
+     */
+    delete(id, options) {
+        return this._client.delete((0, path_1.path) `/ai/tests/test_framework/v1/metrics/${id}`, options);
+    }
+    /**
+     * Returns the specified metric.
+     *
+     * @example
+     * ```ts
+     * const aiTestMetric =
+     *   await client.ai.tests.testFramework.v1.metrics.retrieve(
+     *     0,
+     *   );
+     * ```
+     */
+    retrieve(id, options) {
+        return this._client.get((0, path_1.path) `/ai/tests/test_framework/v1/metrics/${id}`, options);
+    }
+    /**
+     * Updates the specified metric.
+     *
+     * @example
+     * ```ts
+     * const aiTestMetric =
+     *   await client.ai.tests.testFramework.v1.metrics.update(0, {
+     *     audio_enabled: true,
+     *     name: 'Escalates on refund denial',
+     *     prompt:
+     *       "Score the assistant's escalation behaviour between 0.0 and 1.0, considering tone.",
+     *   });
+     * ```
+     */
+    update(id, body, options) {
+        return this._client.patch((0, path_1.path) `/ai/tests/test_framework/v1/metrics/${id}`, { body, ...options });
+    }
+}
+exports.Metrics = Metrics;
+//# sourceMappingURL=metrics.js.map

package/resources/ai/tests/test-framework/v1/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../../../src/resources/ai/tests/test-framework/v1/metrics.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,8DAA2D;AAG3D,gEAA0D;AAE1D;;GAEG;AACH,MAAa,OAAQ,SAAQ,sBAAW;IACtC;;;;;;;;;OASG;IACH,IAAI,CACF,QAA6C,EAAE,EAC/C,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,qCAAqC,EAAE,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,IAAwB,EAAE,OAAwB;QACvD,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,qCAAqC,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;IAED;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,MAA8B,EAAE,OAAwB;QACjE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,iDAAiD,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC1G,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,eAAe,CACb,IAAiC,EACjC,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,sDAAsD,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACzG,CAAC;IAED;;;;;;;;;;;OAWG;IACH,+BAA+B,CAC7B,KAAkD,EAClD,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,+DAA+D,EAAE;YACvF,KAAK;YACL,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,MAAM,CAAC,EAAU,EAAE,OAAwB;QACzC,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAA,WAAI,EAAA,uCAAuC,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;IACvF,CAAC;IAED;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAU,EAAE,OAAwB;QAC3C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,uCAAuC,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;IACpF,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,EAAU,EAAE,IAAwB,EAAE,OAAwB;QACnE,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAA,WAAI,EAAA,uCAAuC,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnG,CAAC;CACF;AArJD,0BAqJC"}

package/resources/ai/tests/test-framework/v1/metrics.mjs

@@ -0,0 +1,141 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+import { APIResource } from "../../../../../core/resource.mjs";
+import { path } from "../../../../../internal/utils/path.mjs";
+/**
+ * Test agents, evaluator scenarios, metrics, and run results for the AI test framework.
+ */
+export class Metrics extends APIResource {
+    /**
+     * Returns the metrics available in your organization, optionally filtered by test
+     * agent. Each metric scores one aspect of a conversation or scenario run.
+     *
+     * @example
+     * ```ts
+     * const aiTestMetrics =
+     *   await client.ai.tests.testFramework.v1.metrics.list();
+     * ```
+     */
+    list(query = {}, options) {
+        return this._client.get('/ai/tests/test_framework/v1/metrics', { query, ...options });
+    }
+    /**
+     * Creates a metric in your organization.
+     *
+     * @example
+     * ```ts
+     * const aiTestMetric =
+     *   await client.ai.tests.testFramework.v1.metrics.create({
+     *     name: 'Escalates on refund denial',
+     *     agents: [1259],
+     *     description:
+     *       "Returns a score for whether the assistant escalates when the caller's refund is denied.",
+     *     eval_type: 'continuous_qualitative',
+     *     observability_enabled: true,
+     *     prompt:
+     *       "Score the assistant's escalation behaviour between 0.0 and 1.0.",
+     *     simulation_enabled: true,
+     *     type: 'llm_judge',
+     *   });
+     * ```
+     */
+    create(body, options) {
+        return this._client.post('/ai/tests/test_framework/v1/metrics', { body, ...options });
+    }
+    /**
+     * Creates several metrics in one request. Accepts an array of metric objects and
+     * returns the created metrics.
+     *
+     * @example
+     * ```ts
+     * const aiTestMetrics =
+     *   await client.ai.tests.testFramework.v1.metrics.bulkCreate(
+     *     { body: [{ name: 'Greets within 5 seconds' }] },
+     *   );
+     * ```
+     */
+    bulkCreate(params, options) {
+        const { body } = params;
+        return this._client.post('/ai/tests/test_framework/v1/metrics/bulk_create', { body: body, ...options });
+    }
+    /**
+     * Starts background generation of suggested metrics for a test agent. Returns a
+     * progress ID to poll with the get-test-metric-generation-progress endpoint.
+     *
+     * @example
+     * ```ts
+     * const scenarioGenerationResponse =
+     *   await client.ai.tests.testFramework.v1.metrics.generateMetrics(
+     *     {
+     *       agent_id: 1259,
+     *       num_metrics: 5,
+     *       scenarios: [9512],
+     *     },
+     *   );
+     * ```
+     */
+    generateMetrics(body, options) {
+        return this._client.post('/ai/tests/test_framework/v1/metrics/generate_metrics', { body, ...options });
+    }
+    /**
+     * Returns the status of a background metric-generation job started with the
+     * generate-test-metrics endpoint.
+     *
+     * @example
+     * ```ts
+     * const response =
+     *   await client.ai.tests.testFramework.v1.metrics.retrieveGenerateMetricsProgress(
+     *     { progress_id: 'progress_id' },
+     *   );
+     * ```
+     */
+    retrieveGenerateMetricsProgress(query, options) {
+        return this._client.get('/ai/tests/test_framework/v1/metrics/generate_metrics_progress', {
+            query,
+            ...options,
+        });
+    }
+    /**
+     * Deletes the specified metric.
+     *
+     * @example
+     * ```ts
+     * const metric =
+     *   await client.ai.tests.testFramework.v1.metrics.delete(0);
+     * ```
+     */
+    delete(id, options) {
+        return this._client.delete(path `/ai/tests/test_framework/v1/metrics/${id}`, options);
+    }
+    /**
+     * Returns the specified metric.
+     *
+     * @example
+     * ```ts
+     * const aiTestMetric =
+     *   await client.ai.tests.testFramework.v1.metrics.retrieve(
+     *     0,
+     *   );
+     * ```
+     */
+    retrieve(id, options) {
+        return this._client.get(path `/ai/tests/test_framework/v1/metrics/${id}`, options);
+    }
+    /**
+     * Updates the specified metric.
+     *
+     * @example
+     * ```ts
+     * const aiTestMetric =
+     *   await client.ai.tests.testFramework.v1.metrics.update(0, {
+     *     audio_enabled: true,
+     *     name: 'Escalates on refund denial',
+     *     prompt:
+     *       "Score the assistant's escalation behaviour between 0.0 and 1.0, considering tone.",
+     *   });
+     * ```
+     */
+    update(id, body, options) {
+        return this._client.patch(path `/ai/tests/test_framework/v1/metrics/${id}`, { body, ...options });
+    }
+}
+//# sourceMappingURL=metrics.mjs.map

package/resources/ai/tests/test-framework/v1/metrics.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.mjs","sourceRoot":"","sources":["../../../../../src/resources/ai/tests/test-framework/v1/metrics.ts"],"names":[],"mappings":"AAAA,sFAAsF;AAEtF,OAAO,EAAE,WAAW,EAAE,yCAAqC;AAG3D,OAAO,EAAE,IAAI,EAAE,+CAA2C;AAE1D;;GAEG;AACH,MAAM,OAAO,OAAQ,SAAQ,WAAW;IACtC;;;;;;;;;OASG;IACH,IAAI,CACF,QAA6C,EAAE,EAC/C,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,qCAAqC,EAAE,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,IAAwB,EAAE,OAAwB;QACvD,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,qCAAqC,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;IAED;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,MAA8B,EAAE,OAAwB;QACjE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,iDAAiD,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC1G,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,eAAe,CACb,IAAiC,EACjC,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,sDAAsD,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACzG,CAAC;IAED;;;;;;;;;;;OAWG;IACH,+BAA+B,CAC7B,KAAkD,EAClD,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,+DAA+D,EAAE;YACvF,KAAK;YACL,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,MAAM,CAAC,EAAU,EAAE,OAAwB;QACzC,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAA,uCAAuC,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;IACvF,CAAC;IAED;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAU,EAAE,OAAwB;QAC3C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,uCAAuC,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;IACpF,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,EAAU,EAAE,IAAwB,EAAE,OAAwB;QACnE,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAA,uCAAuC,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnG,CAAC;CACF"}