scorecard-ai 1.0.0-alpha.7 → 1.0.0-alpha.8

package/resources/testsets.mjs

@@ -13,7 +13,7 @@ export class Testsets extends APIResource {
      *   description: 'Testset for long context Q&A chatbot.',
      *   fieldMapping: {
      *     inputs: ['question'],
-     *     labels: ['idealAnswer'],
+     *     expected: ['idealAnswer'],
      *     metadata: [],
      *   },
      *   jsonSchema: {

package/src/client.ts

@@ -30,7 +30,7 @@ import {
   ProjectsPaginatedResponse,
 } from './resources/projects';
 import { Record as RecordsAPIRecord, RecordCreateParams, Records } from './resources/records';
-import { Run, RunCreateParams, RunUpdateParams, RunUpdateResponse, Runs } from './resources/runs';
+import { Run, RunCreateParams, Runs } from './resources/runs';
 import { Score, ScoreUpsertParams, Scores } from './resources/scores';
 import {
   SystemConfig,
@@ -840,13 +840,7 @@ export declare namespace Scorecard {
     type TestcaseDeleteParams as TestcaseDeleteParams,
   };
 
-  export {
-    Runs as Runs,
-    type Run as Run,
-    type RunUpdateResponse as RunUpdateResponse,
-    type RunCreateParams as RunCreateParams,
-    type RunUpdateParams as RunUpdateParams,
-  };
+  export { Runs as Runs, type Run as Run, type RunCreateParams as RunCreateParams };
 
   export {
     Records as Records,

package/src/lib/runAndEvaluate.ts

@@ -35,7 +35,7 @@ export async function runAndEvaluate<SystemInput extends Object, SystemOutput ex
     const promise = scorecard.records.create(run.id, {
       testcaseId: testcase.id,
       inputs: testcase.inputs,
-      labels: testcase.labels,
+      expected: testcase.expected,
       outputs: modelResponse as Record<string, unknown>,
     });
     recordPromises.push(promise);
@@ -43,11 +43,6 @@ export async function runAndEvaluate<SystemInput extends Object, SystemOutput ex
   // Wait until all the Records are created
   await Promise.all(recordPromises);
 
-  // Mark the Run as done with execution and ready for scoring.
-  await scorecard.runs.update(run.id, {
-    status: 'awaiting_scoring',
-  });
-
   const runUrl = `https://app.getscorecard.ai/projects/${projectId}/runs/grades/${run.id}`;
 
   return { id: run.id, url: runUrl };

package/src/resources/index.ts

@@ -9,7 +9,7 @@ export {
   type ProjectsPaginatedResponse,
 } from './projects';
 export { Records, type Record, type RecordCreateParams } from './records';
-export { Runs, type Run, type RunUpdateResponse, type RunCreateParams, type RunUpdateParams } from './runs';
+export { Runs, type Run, type RunCreateParams } from './runs';
 export { Scores, type Score, type ScoreUpsertParams } from './scores';
 export {
   SystemConfigs,

package/src/resources/records.ts

@@ -13,8 +13,10 @@ export class Records extends APIResource {
    * @example
    * ```ts
    * const record = await client.records.create('135', {
+   *   expected: {
+   *     idealAnswer: 'Paris is the capital of France',
+   *   },
    *   inputs: { question: 'What is the capital of France?' },
-   *   labels: { idealAnswer: 'Paris is the capital of France' },
    *   outputs: { response: 'The capital of France is Paris.' },
    *   testcaseId: '248',
    * });
@@ -35,15 +37,15 @@ export interface Record {
   id: string;
 
   /**
-   * The actual inputs sent to the system, which should match the system's input
-   * schema.
+   * The expected outputs for the Testcase.
    */
-  inputs: BuiltinRecord<string, unknown>;
+  expected: BuiltinRecord<string, unknown>;
 
   /**
-   * The expected outputs for the Testcase.
+   * The actual inputs sent to the system, which should match the system's input
+   * schema.
    */
-  labels: BuiltinRecord<string, unknown>;
+  inputs: BuiltinRecord<string, unknown>;
 
   /**
    * The actual outputs from the system.
@@ -63,15 +65,15 @@ export interface Record {
 
 export interface RecordCreateParams {
   /**
-   * The actual inputs sent to the system, which should match the system's input
-   * schema.
+   * The expected outputs for the Testcase.
    */
-  inputs: BuiltinRecord<string, unknown>;
+  expected: BuiltinRecord<string, unknown>;
 
   /**
-   * The expected outputs for the Testcase.
+   * The actual inputs sent to the system, which should match the system's input
+   * schema.
    */
-  labels: BuiltinRecord<string, unknown>;
+  inputs: BuiltinRecord<string, unknown>;
 
   /**
    * The actual outputs from the system.

package/src/resources/runs.ts

@@ -21,20 +21,6 @@ export class Runs extends APIResource {
   create(projectID: string, body: RunCreateParams, options?: RequestOptions): APIPromise<Run> {
     return this._client.post(path`/projects/${projectID}/runs`, { body, ...options });
   }
-
-  /**
-   * Update the status of a Run.
-   *
-   * @example
-   * ```ts
-   * const run = await client.runs.update('135', {
-   *   status: 'awaiting_scoring',
-   * });
-   * ```
-   */
-  update(runID: string, body: RunUpdateParams, options?: RequestOptions): APIPromise<RunUpdateResponse> {
-    return this._client.patch(path`/runs/${runID}`, { body, ...options });
-  }
 }
 
 /**
@@ -74,25 +60,6 @@ export interface Run {
   systemConfigId?: string;
 }
 
-export interface RunUpdateResponse {
-  /**
-   * The ID of the Run.
-   */
-  id: string;
-
-  /**
-   * The status of the Run.
-   */
-  status:
-    | 'pending'
-    | 'awaiting_execution'
-    | 'running_execution'
-    | 'awaiting_scoring'
-    | 'running_scoring'
-    | 'awaiting_human_scoring'
-    | 'completed';
-}
-
 export interface RunCreateParams {
   /**
    * The IDs of the metrics this Run is using.
@@ -110,25 +77,6 @@ export interface RunCreateParams {
   systemConfigId?: string;
 }
 
-export interface RunUpdateParams {
-  /**
-   * The status of the Run.
-   */
-  status:
-    | 'pending'
-    | 'awaiting_execution'
-    | 'running_execution'
-    | 'awaiting_scoring'
-    | 'running_scoring'
-    | 'awaiting_human_scoring'
-    | 'completed';
-}
-
 export declare namespace Runs {
-  export {
-    type Run as Run,
-    type RunUpdateResponse as RunUpdateResponse,
-    type RunCreateParams as RunCreateParams,
-    type RunUpdateParams as RunUpdateParams,
-  };
+  export { type Run as Run, type RunCreateParams as RunCreateParams };
 }

package/src/resources/testcases.ts

@@ -120,11 +120,11 @@ export type TestcasesPaginatedResponse = PaginatedResponse<Testcase>;
 
 /**
  * A test case in the Scorecard system. Contains JSON data that is validated
- * against the schema defined by its Testset. The `inputs` and `labels` fields are
- * derived from the `data` field based on the Testset's `fieldMapping`, and include
- * all mapped fields, including those with validation errors. Testcases are stored
- * regardless of validation results, with any validation errors included in the
- * `validationErrors` field.
+ * against the schema defined by its Testset. The `inputs` and `expected` fields
+ * are derived from the `data` field based on the Testset's `fieldMapping`, and
+ * include all mapped fields, including those with validation errors. Testcases are
+ * stored regardless of validation results, with any validation errors included in
+ * the `validationErrors` field.
  */
 export interface Testcase {
   /**
@@ -132,6 +132,12 @@ export interface Testcase {
    */
   id: string;
 
+  /**
+   * Derived from data based on the Testset's fieldMapping. Contains all fields
+   * marked as expected outputs, including those with validation errors.
+   */
+  expected: Record<string, unknown>;
+
   /**
    * Derived from data based on the Testset's fieldMapping. Contains all fields
    * marked as inputs, including those with validation errors.
@@ -143,12 +149,6 @@ export interface Testcase {
    */
   jsonData: Record<string, unknown>;
 
-  /**
-   * Derived from data based on the Testset's fieldMapping. Contains all fields
-   * marked as labels, including those with validation errors.
-   */
-  labels: Record<string, unknown>;
-
   /**
    * The ID of the Testset this Testcase belongs to.
    */
@@ -194,14 +194,6 @@ export interface TestcaseCreateParams {
 }
 
 export namespace TestcaseCreateParams {
-  /**
-   * A test case in the Scorecard system. Contains JSON data that is validated
-   * against the schema defined by its Testset. The `inputs` and `labels` fields are
-   * derived from the `data` field based on the Testset's `fieldMapping`, and include
-   * all mapped fields, including those with validation errors. Testcases are stored
-   * regardless of validation results, with any validation errors included in the
-   * `validationErrors` field.
-   */
   export interface Item {
     /**
      * The JSON data of the Testcase, which is validated against the Testset's schema.

package/src/resources/testsets.ts

@@ -17,7 +17,7 @@ export class Testsets extends APIResource {
    *   description: 'Testset for long context Q&A chatbot.',
    *   fieldMapping: {
    *     inputs: ['question'],
-   *     labels: ['idealAnswer'],
+   *     expected: ['idealAnswer'],
    *     metadata: [],
    *   },
    *   jsonSchema: {
@@ -119,9 +119,9 @@ export type TestsetsPaginatedResponse = PaginatedResponse<Testset>;
 /**
  * A collection of Testcases that share the same schema. Each Testset defines the
  * structure of its Testcases through a JSON schema. The `fieldMapping` object maps
- * top-level keys of the Testcase schema to their roles (input/label). Fields not
- * mentioned in the `fieldMapping` during creation or update are treated as
- * metadata.
+ * top-level keys of the Testcase schema to their roles (input/expected output).
+ * Fields not mentioned in the `fieldMapping` during creation or update are treated
+ * as metadata.
  *
  * ## JSON Schema validation constraints supported:
  *
@@ -154,8 +154,8 @@ export interface Testset {
   description: string;
 
   /**
-   * Maps top-level keys of the Testcase schema to their roles (input/label).
-   * Unmapped fields are treated as metadata.
+   * Maps top-level keys of the Testcase schema to their roles (input/expected
+   * output). Unmapped fields are treated as metadata.
    */
   fieldMapping: Testset.FieldMapping;
 
@@ -172,22 +172,22 @@ export interface Testset {
 
 export namespace Testset {
   /**
-   * Maps top-level keys of the Testcase schema to their roles (input/label).
-   * Unmapped fields are treated as metadata.
+   * Maps top-level keys of the Testcase schema to their roles (input/expected
+   * output). Unmapped fields are treated as metadata.
    */
   export interface FieldMapping {
     /**
-     * Fields that represent inputs to the AI system.
+     * Fields that represent expected outputs.
      */
-    inputs: Array<string>;
+    expected: Array<string>;
 
     /**
-     * Fields that represent expected outputs/labels.
+     * Fields that represent inputs to the AI system.
      */
-    labels: Array<string>;
+    inputs: Array<string>;
 
     /**
-     * Fields that are not inputs or labels.
+     * Fields that are not inputs or expected outputs.
      */
     metadata: Array<string>;
   }
@@ -207,8 +207,8 @@ export interface TestsetCreateParams {
   description: string;
 
   /**
-   * Maps top-level keys of the Testcase schema to their roles (input/label).
-   * Unmapped fields are treated as metadata.
+   * Maps top-level keys of the Testcase schema to their roles (input/expected
+   * output). Unmapped fields are treated as metadata.
    */
   fieldMapping: TestsetCreateParams.FieldMapping;
 
@@ -225,22 +225,22 @@ export interface TestsetCreateParams {
 
 export namespace TestsetCreateParams {
   /**
-   * Maps top-level keys of the Testcase schema to their roles (input/label).
-   * Unmapped fields are treated as metadata.
+   * Maps top-level keys of the Testcase schema to their roles (input/expected
+   * output). Unmapped fields are treated as metadata.
    */
   export interface FieldMapping {
     /**
-     * Fields that represent inputs to the AI system.
+     * Fields that represent expected outputs.
      */
-    inputs: Array<string>;
+    expected: Array<string>;
 
     /**
-     * Fields that represent expected outputs/labels.
+     * Fields that represent inputs to the AI system.
      */
-    labels: Array<string>;
+    inputs: Array<string>;
 
     /**
-     * Fields that are not inputs or labels.
+     * Fields that are not inputs or expected outputs.
      */
     metadata: Array<string>;
   }
@@ -253,8 +253,8 @@ export interface TestsetUpdateParams {
   description?: string;
 
   /**
-   * Maps top-level keys of the Testcase schema to their roles (input/label).
-   * Unmapped fields are treated as metadata.
+   * Maps top-level keys of the Testcase schema to their roles (input/expected
+   * output). Unmapped fields are treated as metadata.
    */
   fieldMapping?: TestsetUpdateParams.FieldMapping;
 
@@ -271,22 +271,22 @@ export interface TestsetUpdateParams {
 
 export namespace TestsetUpdateParams {
   /**
-   * Maps top-level keys of the Testcase schema to their roles (input/label).
-   * Unmapped fields are treated as metadata.
+   * Maps top-level keys of the Testcase schema to their roles (input/expected
+   * output). Unmapped fields are treated as metadata.
    */
   export interface FieldMapping {
     /**
-     * Fields that represent inputs to the AI system.
+     * Fields that represent expected outputs.
      */
-    inputs: Array<string>;
+    expected: Array<string>;
 
     /**
-     * Fields that represent expected outputs/labels.
+     * Fields that represent inputs to the AI system.
      */
-    labels: Array<string>;
+    inputs: Array<string>;
 
     /**
-     * Fields that are not inputs or labels.
+     * Fields that are not inputs or expected outputs.
      */
     metadata: Array<string>;
   }

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = '1.0.0-alpha.7'; // x-release-please-version
+export const VERSION = '1.0.0-alpha.8'; // x-release-please-version

package/version.d.mts

@@ -1,2 +1,2 @@
-export declare const VERSION = "1.0.0-alpha.7";
+export declare const VERSION = "1.0.0-alpha.8";
 //# sourceMappingURL=version.d.mts.map

package/version.d.ts

@@ -1,2 +1,2 @@
-export declare const VERSION = "1.0.0-alpha.7";
+export declare const VERSION = "1.0.0-alpha.8";
 //# sourceMappingURL=version.d.ts.map

package/version.js

@@ -1,5 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = '1.0.0-alpha.7'; // x-release-please-version
+exports.VERSION = '1.0.0-alpha.8'; // x-release-please-version
 //# sourceMappingURL=version.js.map

package/version.mjs

@@ -1,2 +1,2 @@
-export const VERSION = '1.0.0-alpha.7'; // x-release-please-version
+export const VERSION = '1.0.0-alpha.8'; // x-release-please-version
 //# sourceMappingURL=version.mjs.map