scorecard-ai 1.0.0-alpha.7 → 1.0.0-alpha.9

package/src/internal/detect-platform.ts

@@ -85,10 +85,10 @@ const getPlatformProperties = (): PlatformProperties => {
     return {
       'X-Stainless-Lang': 'js',
       'X-Stainless-Package-Version': VERSION,
-      'X-Stainless-OS': normalizePlatform((globalThis as any).process.platform),
-      'X-Stainless-Arch': normalizeArch((globalThis as any).process.arch),
+      'X-Stainless-OS': normalizePlatform((globalThis as any).process.platform ?? 'unknown'),
+      'X-Stainless-Arch': normalizeArch((globalThis as any).process.arch ?? 'unknown'),
       'X-Stainless-Runtime': 'node',
-      'X-Stainless-Runtime-Version': (globalThis as any).process.version,
+      'X-Stainless-Runtime-Version': (globalThis as any).process.version ?? 'unknown',
     };
   }
 

package/src/internal/shim-types.ts

@@ -0,0 +1,26 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+/**
+ * Shims for types that we can't always rely on being available globally.
+ *
+ * Note: these only exist at the type-level, there is no corresponding runtime
+ * version for any of these symbols.
+ */
+
+type NeverToAny<T> = T extends never ? any : T;
+
+/** @ts-ignore */
+type _DOMReadableStream<R = any> = globalThis.ReadableStream<R>;
+
+/** @ts-ignore */
+type _NodeReadableStream<R = any> = import('stream/web').ReadableStream<R>;
+
+type _ConditionalNodeReadableStream<R = any> =
+  typeof globalThis extends { ReadableStream: any } ? never : _NodeReadableStream<R>;
+
+type _ReadableStream<R = any> = NeverToAny<
+  | ([0] extends [1 & _DOMReadableStream<R>] ? never : _DOMReadableStream<R>)
+  | ([0] extends [1 & _ConditionalNodeReadableStream<R>] ? never : _ConditionalNodeReadableStream<R>)
+>;
+
+export type { _ReadableStream as ReadableStream };

package/src/internal/shims.ts

@@ -7,8 +7,8 @@
  * messages in cases where an environment isn't fully supported.
  */
 
-import { type Fetch } from './builtin-types';
-import { type ReadableStream } from './shim-types';
+import type { Fetch } from './builtin-types';
+import type { ReadableStream } from './shim-types';
 
 export function getDefaultFetch(): Fetch {
   if (typeof fetch !== 'undefined') {

package/src/internal/uploads.ts

@@ -138,7 +138,7 @@ export const createForm = async <T = Record<string, unknown>>(
 
 // We check for Blob not File because Bun.File doesn't inherit from File,
 // but they both inherit from Blob and have a `name` property at runtime.
-const isNamedBlob = (value: object) => value instanceof Blob && 'name' in value;
+const isNamedBlob = (value: unknown) => value instanceof Blob && 'name' in value;
 
 const isUploadable = (value: unknown) =>
   typeof value === 'object' &&

package/src/lib/runAndEvaluate.ts

@@ -1,54 +1,152 @@
 import { Scorecard } from '../client';
+import { SystemConfig, Testcase } from '../resources';
+
+type RunAndEvaluateArgs<SystemInput extends Record<string, any>, SystemOutput extends Record<string, any>> =
+  // Project and metrics are always required
+  {
+    /**
+     * The ID of the Project to run the system on.
+     */
+    projectId: string;
+
+    /**
+     * The IDs of the Metrics to use for evaluation.
+     */
+    metricIds: Array<string>;
+  } & (
+    | // If system config is provided, the system function receives a system config
+    {
+        /**
+         * The ID of the System Configuration to use for the run.
+         */
+        systemConfigId: string;
+
+        /**
+         * The system function to run on the Testset.
+         */
+        system: (testcaseInput: SystemInput, systemConfig: SystemConfig) => Promise<SystemOutput>;
+      }
+    // Otherwise, the system function receives only the testcase input
+    | {
+        /**
+         * The system function to run on the Testset.
+         */
+        system: (testcaseInput: SystemInput) => Promise<SystemOutput>;
+      }
+  ) &
+    // If testset is not provided, you must pass in all the testcases manually
+    (| {
+          /**
+           * The ID of the Scorecard Testset to run the system on.
+           */
+          testsetId: string;
+        }
+      | {
+          /**
+           * The list of test cases to run the system on. Can be a list of Scorecard Testcases or a list of inputs and expected outputs.
+           */
+          testcases:
+            | Array<{
+                inputs: SystemInput;
+                expected: Record<string, unknown>;
+              }>
+            | Array<Testcase>;
+        }
+    );
+
+/**
+ * Returns an async generator over the given Testset or Testcases.
+ */
+async function* testcaseIterator<SystemInput extends Record<string, any>>(
+  scorecard: Scorecard,
+  args: RunAndEvaluateArgs<SystemInput, any>,
+): AsyncGenerator<{
+  testcaseId: string | null;
+  inputs: SystemInput;
+  expected: Record<string, unknown>;
+}> {
+  if ('testsetId' in args) {
+    for await (const testcase of scorecard.testcases.list(args.testsetId)) {
+      yield {
+        ...testcase,
+        testcaseId: testcase.id,
+        inputs: testcase.inputs as SystemInput,
+      };
+    }
+  } else {
+    for (const testcase of args.testcases) {
+      yield {
+        ...testcase,
+        testcaseId: 'id' in testcase ? testcase.id : null,
+        inputs: testcase.inputs as SystemInput,
+      };
+    }
+  }
+}
 
 /**
  * Runs a system on a Testset and records the results in Scorecard.
  *
  * @param scorecard The Scorecard client
- * @param projectId The ID of the Project to run the system on.
- * @param testsetId The ID of the Testset to run the system on.
- * @param metricIds The IDs of the Metrics to use for evaluation.
- * @param system The system to run on the Testset.
+ * @param args.projectId The ID of the Project to run the system on.
+ * @param args.testsetId The optional ID of the Testset to run the system on. Either this or `args.testcases` must be provided.
+ * @param args.testcases The optional list of Testcases to run the system on. Either this or `args.testsetId` must be provided.
+ * @param args.metricIds The IDs of the Metrics to use for evaluation.
+ * @param args.systemConfigId The optional ID of the System Configuration to associate with the Run.
+ * @param args.system The system to run on the Testset.
+ * @param options.runInParallel Whether to call `args.system` in parallel. False (sequential) by default.
  */
-export async function runAndEvaluate<SystemInput extends Object, SystemOutput extends Object>(
+export async function runAndEvaluate<
+  SystemInput extends Record<string, any>,
+  SystemOutput extends Record<string, any>,
+>(
   scorecard: Scorecard,
-  {
-    projectId,
-    testsetId,
-    metricIds,
-    system,
-  }: {
-    projectId: string;
-    testsetId: string;
-    metricIds: Array<string>;
-    system: (testcaseInput: SystemInput) => Promise<SystemOutput>;
+  args: RunAndEvaluateArgs<SystemInput, SystemOutput>,
+  options: {
+    runInParallel: boolean;
+  } = {
+    runInParallel: false,
   },
 ): Promise<Pick<Scorecard.Runs.Run, 'id'> & { url: string }> {
-  const run = await scorecard.runs.create(projectId, {
-    testsetId,
-    metricIds,
+  const hasSystemConfig = 'systemConfigId' in args;
+  const hasTestset = 'testsetId' in args;
+
+  const runPromise = scorecard.runs.create(args.projectId, {
+    testsetId: hasTestset ? args.testsetId : null,
+    metricIds: args.metricIds,
+    ...(hasSystemConfig ?
+      {
+        systemConfigId: args.systemConfigId,
+      }
+    : null),
   });
+  const systemConfig = hasSystemConfig ? await scorecard.systemConfigs.get(args.systemConfigId) : null;
+  const run = await runPromise;
 
-  // Run each Testcase sequentially
-  const recordPromises: Array<Promise<any>> = [];
-  for await (const testcase of scorecard.testcases.list(run.testsetId)) {
-    const modelResponse = await system(testcase.inputs as SystemInput);
-    const promise = scorecard.records.create(run.id, {
-      testcaseId: testcase.id,
-      inputs: testcase.inputs,
-      labels: testcase.labels,
-      outputs: modelResponse as Record<string, unknown>,
-    });
-    recordPromises.push(promise);
+  const recordPromises: Array<Promise<unknown>> = [];
+
+  for await (const { testcaseId, inputs, expected } of testcaseIterator(scorecard, args)) {
+    const modelResponsePromise = hasSystemConfig ? args.system(inputs, systemConfig!) : args.system(inputs);
+
+    function createRecord(outputs: SystemOutput): Promise<unknown> {
+      return scorecard.records.create(run.id, {
+        inputs,
+        expected,
+        outputs,
+        ...(testcaseId != null ? { testcaseId } : null),
+      });
+    }
+
+    if (options.runInParallel) {
+      recordPromises.push(modelResponsePromise.then(createRecord));
+    } else {
+      recordPromises.push(createRecord(await modelResponsePromise));
+    }
   }
   // Wait until all the Records are created
   await Promise.all(recordPromises);
 
-  // Mark the Run as done with execution and ready for scoring.
-  await scorecard.runs.update(run.id, {
-    status: 'awaiting_scoring',
-  });
-
-  const runUrl = `https://app.getscorecard.ai/projects/${projectId}/runs/grades/${run.id}`;
+  const runUrl = `${scorecard.baseAppURL}/projects/${args.projectId}/runs/${run.id}`;
 
   return { id: run.id, url: runUrl };
 }

package/src/resources/index.ts

@@ -9,14 +9,13 @@ export {
   type ProjectsPaginatedResponse,
 } from './projects';
 export { Records, type Record, type RecordCreateParams } from './records';
-export { Runs, type Run, type RunUpdateResponse, type RunCreateParams, type RunUpdateParams } from './runs';
+export { Runs, type Run, type RunCreateParams } from './runs';
 export { Scores, type Score, type ScoreUpsertParams } from './scores';
 export {
   SystemConfigs,
   type SystemConfig,
   type SystemConfigCreateParams,
   type SystemConfigListParams,
-  type SystemConfigGetParams,
   type SystemConfigsPaginatedResponse,
 } from './system-configs';
 export {

package/src/resources/records.ts

@@ -13,8 +13,10 @@ export class Records extends APIResource {
    * @example
    * ```ts
    * const record = await client.records.create('135', {
+   *   expected: {
+   *     idealAnswer: 'Paris is the capital of France',
+   *   },
    *   inputs: { question: 'What is the capital of France?' },
-   *   labels: { idealAnswer: 'Paris is the capital of France' },
    *   outputs: { response: 'The capital of France is Paris.' },
    *   testcaseId: '248',
    * });
@@ -35,15 +37,15 @@ export interface Record {
   id: string;
 
   /**
-   * The actual inputs sent to the system, which should match the system's input
-   * schema.
+   * The expected outputs for the Testcase.
    */
-  inputs: BuiltinRecord<string, unknown>;
+  expected: BuiltinRecord<string, unknown>;
 
   /**
-   * The expected outputs for the Testcase.
+   * The actual inputs sent to the system, which should match the system's input
+   * schema.
    */
-  labels: BuiltinRecord<string, unknown>;
+  inputs: BuiltinRecord<string, unknown>;
 
   /**
    * The actual outputs from the system.
@@ -63,15 +65,15 @@ export interface Record {
 
 export interface RecordCreateParams {
   /**
-   * The actual inputs sent to the system, which should match the system's input
-   * schema.
+   * The expected outputs for the Testcase.
    */
-  inputs: BuiltinRecord<string, unknown>;
+  expected: BuiltinRecord<string, unknown>;
 
   /**
-   * The expected outputs for the Testcase.
+   * The actual inputs sent to the system, which should match the system's input
+   * schema.
    */
-  labels: BuiltinRecord<string, unknown>;
+  inputs: BuiltinRecord<string, unknown>;
 
   /**
    * The actual outputs from the system.

package/src/resources/runs.ts

@@ -13,28 +13,14 @@ export class Runs extends APIResource {
    * ```ts
    * const run = await client.runs.create('314', {
    *   metricIds: ['789', '101'],
-   *   testsetId: '246',
    *   systemConfigId: '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
+   *   testsetId: '246',
    * });
    * ```
    */
   create(projectID: string, body: RunCreateParams, options?: RequestOptions): APIPromise<Run> {
     return this._client.post(path`/projects/${projectID}/runs`, { body, ...options });
   }
-
-  /**
-   * Update the status of a Run.
-   *
-   * @example
-   * ```ts
-   * const run = await client.runs.update('135', {
-   *   status: 'awaiting_scoring',
-   * });
-   * ```
-   */
-  update(runID: string, body: RunUpdateParams, options?: RequestOptions): APIPromise<RunUpdateResponse> {
-    return this._client.patch(path`/runs/${runID}`, { body, ...options });
-  }
 }
 
 /**
@@ -66,7 +52,7 @@ export interface Run {
   /**
    * The ID of the Testset this Run is testing.
    */
-  testsetId: string;
+  testsetId: string | null;
 
   /**
    * The ID of the system configuration this Run is using.
@@ -74,61 +60,23 @@ export interface Run {
   systemConfigId?: string;
 }
 
-export interface RunUpdateResponse {
-  /**
-   * The ID of the Run.
-   */
-  id: string;
-
-  /**
-   * The status of the Run.
-   */
-  status:
-    | 'pending'
-    | 'awaiting_execution'
-    | 'running_execution'
-    | 'awaiting_scoring'
-    | 'running_scoring'
-    | 'awaiting_human_scoring'
-    | 'completed';
-}
-
 export interface RunCreateParams {
   /**
    * The IDs of the metrics this Run is using.
    */
   metricIds: Array<string>;
 
-  /**
-   * The ID of the Testset this Run is testing.
-   */
-  testsetId: string;
-
   /**
    * The ID of the system configuration this Run is using.
    */
   systemConfigId?: string;
-}
 
-export interface RunUpdateParams {
   /**
-   * The status of the Run.
+   * The ID of the Testset this Run is testing.
    */
-  status:
-    | 'pending'
-    | 'awaiting_execution'
-    | 'running_execution'
-    | 'awaiting_scoring'
-    | 'running_scoring'
-    | 'awaiting_human_scoring'
-    | 'completed';
+  testsetId?: string | null;
 }
 
 export declare namespace Runs {
-  export {
-    type Run as Run,
-    type RunUpdateResponse as RunUpdateResponse,
-    type RunCreateParams as RunCreateParams,
-    type RunUpdateParams as RunUpdateParams,
-  };
+  export { type Run as Run, type RunCreateParams as RunCreateParams };
 }

package/src/resources/system-configs.ts

@@ -81,17 +81,11 @@ export class SystemConfigs extends APIResource {
    * ```ts
    * const systemConfig = await client.systemConfigs.get(
    *   '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
-   *   { systemId: '12345678-0a8b-4f66-b6f3-2ddcfa097257' },
    * );
    * ```
    */
-  get(
-    systemConfigID: string,
-    params: SystemConfigGetParams,
-    options?: RequestOptions,
-  ): APIPromise<SystemConfig> {
-    const { systemId } = params;
-    return this._client.get(path`/systems/${systemId}/configs/${systemConfigID}`, options);
+  get(systemConfigID: string, options?: RequestOptions): APIPromise<SystemConfig> {
+    return this._client.get(path`/systems/configs/${systemConfigID}`, options);
   }
 }
 
@@ -185,19 +179,11 @@ export namespace SystemConfigCreateParams {
 
 export interface SystemConfigListParams extends PaginatedResponseParams {}
 
-export interface SystemConfigGetParams {
-  /**
-   * The ID of the system the configuration belongs to.
-   */
-  systemId: string;
-}
-
 export declare namespace SystemConfigs {
   export {
     type SystemConfig as SystemConfig,
     type SystemConfigsPaginatedResponse as SystemConfigsPaginatedResponse,
     type SystemConfigCreateParams as SystemConfigCreateParams,
     type SystemConfigListParams as SystemConfigListParams,
-    type SystemConfigGetParams as SystemConfigGetParams,
   };
 }

package/src/resources/testcases.ts

@@ -120,11 +120,11 @@ export type TestcasesPaginatedResponse = PaginatedResponse<Testcase>;
 
 /**
  * A test case in the Scorecard system. Contains JSON data that is validated
- * against the schema defined by its Testset. The `inputs` and `labels` fields are
- * derived from the `data` field based on the Testset's `fieldMapping`, and include
- * all mapped fields, including those with validation errors. Testcases are stored
- * regardless of validation results, with any validation errors included in the
- * `validationErrors` field.
+ * against the schema defined by its Testset. The `inputs` and `expected` fields
+ * are derived from the `data` field based on the Testset's `fieldMapping`, and
+ * include all mapped fields, including those with validation errors. Testcases are
+ * stored regardless of validation results, with any validation errors included in
+ * the `validationErrors` field.
  */
 export interface Testcase {
   /**
@@ -132,6 +132,12 @@ export interface Testcase {
    */
   id: string;
 
+  /**
+   * Derived from data based on the Testset's fieldMapping. Contains all fields
+   * marked as expected outputs, including those with validation errors.
+   */
+  expected: Record<string, unknown>;
+
   /**
    * Derived from data based on the Testset's fieldMapping. Contains all fields
    * marked as inputs, including those with validation errors.
@@ -143,12 +149,6 @@ export interface Testcase {
    */
   jsonData: Record<string, unknown>;
 
-  /**
-   * Derived from data based on the Testset's fieldMapping. Contains all fields
-   * marked as labels, including those with validation errors.
-   */
-  labels: Record<string, unknown>;
-
   /**
    * The ID of the Testset this Testcase belongs to.
    */
@@ -194,14 +194,6 @@ export interface TestcaseCreateParams {
 }
 
 export namespace TestcaseCreateParams {
-  /**
-   * A test case in the Scorecard system. Contains JSON data that is validated
-   * against the schema defined by its Testset. The `inputs` and `labels` fields are
-   * derived from the `data` field based on the Testset's `fieldMapping`, and include
-   * all mapped fields, including those with validation errors. Testcases are stored
-   * regardless of validation results, with any validation errors included in the
-   * `validationErrors` field.
-   */
   export interface Item {
     /**
      * The JSON data of the Testcase, which is validated against the Testset's schema.

package/src/resources/testsets.ts

@@ -17,7 +17,7 @@ export class Testsets extends APIResource {
    *   description: 'Testset for long context Q&A chatbot.',
    *   fieldMapping: {
    *     inputs: ['question'],
-   *     labels: ['idealAnswer'],
+   *     expected: ['idealAnswer'],
    *     metadata: [],
    *   },
    *   jsonSchema: {
@@ -119,9 +119,9 @@ export type TestsetsPaginatedResponse = PaginatedResponse<Testset>;
 /**
  * A collection of Testcases that share the same schema. Each Testset defines the
  * structure of its Testcases through a JSON schema. The `fieldMapping` object maps
- * top-level keys of the Testcase schema to their roles (input/label). Fields not
- * mentioned in the `fieldMapping` during creation or update are treated as
- * metadata.
+ * top-level keys of the Testcase schema to their roles (input/expected output).
+ * Fields not mentioned in the `fieldMapping` during creation or update are treated
+ * as metadata.
  *
  * ## JSON Schema validation constraints supported:
  *
@@ -154,8 +154,8 @@ export interface Testset {
   description: string;
 
   /**
-   * Maps top-level keys of the Testcase schema to their roles (input/label).
-   * Unmapped fields are treated as metadata.
+   * Maps top-level keys of the Testcase schema to their roles (input/expected
+   * output). Unmapped fields are treated as metadata.
    */
   fieldMapping: Testset.FieldMapping;
 
@@ -172,22 +172,22 @@ export interface Testset {
 
 export namespace Testset {
   /**
-   * Maps top-level keys of the Testcase schema to their roles (input/label).
-   * Unmapped fields are treated as metadata.
+   * Maps top-level keys of the Testcase schema to their roles (input/expected
+   * output). Unmapped fields are treated as metadata.
    */
   export interface FieldMapping {
     /**
-     * Fields that represent inputs to the AI system.
+     * Fields that represent expected outputs.
      */
-    inputs: Array<string>;
+    expected: Array<string>;
 
     /**
-     * Fields that represent expected outputs/labels.
+     * Fields that represent inputs to the AI system.
      */
-    labels: Array<string>;
+    inputs: Array<string>;
 
     /**
-     * Fields that are not inputs or labels.
+     * Fields that are not inputs or expected outputs.
      */
     metadata: Array<string>;
   }
@@ -207,8 +207,8 @@ export interface TestsetCreateParams {
   description: string;
 
   /**
-   * Maps top-level keys of the Testcase schema to their roles (input/label).
-   * Unmapped fields are treated as metadata.
+   * Maps top-level keys of the Testcase schema to their roles (input/expected
+   * output). Unmapped fields are treated as metadata.
    */
   fieldMapping: TestsetCreateParams.FieldMapping;
 
@@ -225,22 +225,22 @@ export interface TestsetCreateParams {
 
 export namespace TestsetCreateParams {
   /**
-   * Maps top-level keys of the Testcase schema to their roles (input/label).
-   * Unmapped fields are treated as metadata.
+   * Maps top-level keys of the Testcase schema to their roles (input/expected
+   * output). Unmapped fields are treated as metadata.
    */
   export interface FieldMapping {
     /**
-     * Fields that represent inputs to the AI system.
+     * Fields that represent expected outputs.
      */
-    inputs: Array<string>;
+    expected: Array<string>;
 
     /**
-     * Fields that represent expected outputs/labels.
+     * Fields that represent inputs to the AI system.
      */
-    labels: Array<string>;
+    inputs: Array<string>;
 
     /**
-     * Fields that are not inputs or labels.
+     * Fields that are not inputs or expected outputs.
      */
     metadata: Array<string>;
   }
@@ -253,8 +253,8 @@ export interface TestsetUpdateParams {
   description?: string;
 
   /**
-   * Maps top-level keys of the Testcase schema to their roles (input/label).
-   * Unmapped fields are treated as metadata.
+   * Maps top-level keys of the Testcase schema to their roles (input/expected
+   * output). Unmapped fields are treated as metadata.
    */
   fieldMapping?: TestsetUpdateParams.FieldMapping;
 
@@ -271,22 +271,22 @@ export interface TestsetUpdateParams {
 
 export namespace TestsetUpdateParams {
   /**
-   * Maps top-level keys of the Testcase schema to their roles (input/label).
-   * Unmapped fields are treated as metadata.
+   * Maps top-level keys of the Testcase schema to their roles (input/expected
+   * output). Unmapped fields are treated as metadata.
    */
   export interface FieldMapping {
     /**
-     * Fields that represent inputs to the AI system.
+     * Fields that represent expected outputs.
      */
-    inputs: Array<string>;
+    expected: Array<string>;
 
     /**
-     * Fields that represent expected outputs/labels.
+     * Fields that represent inputs to the AI system.
      */
-    labels: Array<string>;
+    inputs: Array<string>;
 
     /**
-     * Fields that are not inputs or labels.
+     * Fields that are not inputs or expected outputs.
      */
     metadata: Array<string>;
   }

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = '1.0.0-alpha.7'; // x-release-please-version
+export const VERSION = '1.0.0-alpha.9'; // x-release-please-version

package/version.d.mts

@@ -1,2 +1,2 @@
-export declare const VERSION = "1.0.0-alpha.7";
+export declare const VERSION = "1.0.0-alpha.9";
 //# sourceMappingURL=version.d.mts.map