scorecard-ai 1.0.0-alpha.8 → 1.0.0-alpha.9

package/src/lib/runAndEvaluate.ts

@@ -1,49 +1,152 @@
 import { Scorecard } from '../client';
+import { SystemConfig, Testcase } from '../resources';
+
+type RunAndEvaluateArgs<SystemInput extends Record<string, any>, SystemOutput extends Record<string, any>> =
+  // Project and metrics are always required
+  {
+    /**
+     * The ID of the Project to run the system on.
+     */
+    projectId: string;
+
+    /**
+     * The IDs of the Metrics to use for evaluation.
+     */
+    metricIds: Array<string>;
+  } & (
+    | // If system config is provided, the system function receives a system config
+    {
+        /**
+         * The ID of the System Configuration to use for the run.
+         */
+        systemConfigId: string;
+
+        /**
+         * The system function to run on the Testset.
+         */
+        system: (testcaseInput: SystemInput, systemConfig: SystemConfig) => Promise<SystemOutput>;
+      }
+    // Otherwise, the system function receives only the testcase input
+    | {
+        /**
+         * The system function to run on the Testset.
+         */
+        system: (testcaseInput: SystemInput) => Promise<SystemOutput>;
+      }
+  ) &
+    // If testset is not provided, you must pass in all the testcases manually
+    (| {
+          /**
+           * The ID of the Scorecard Testset to run the system on.
+           */
+          testsetId: string;
+        }
+      | {
+          /**
+           * The list of test cases to run the system on. Can be a list of Scorecard Testcases or a list of inputs and expected outputs.
+           */
+          testcases:
+            | Array<{
+                inputs: SystemInput;
+                expected: Record<string, unknown>;
+              }>
+            | Array<Testcase>;
+        }
+    );
+
+/**
+ * Returns an async generator over the given Testset or Testcases.
+ */
+async function* testcaseIterator<SystemInput extends Record<string, any>>(
+  scorecard: Scorecard,
+  args: RunAndEvaluateArgs<SystemInput, any>,
+): AsyncGenerator<{
+  testcaseId: string | null;
+  inputs: SystemInput;
+  expected: Record<string, unknown>;
+}> {
+  if ('testsetId' in args) {
+    for await (const testcase of scorecard.testcases.list(args.testsetId)) {
+      yield {
+        ...testcase,
+        testcaseId: testcase.id,
+        inputs: testcase.inputs as SystemInput,
+      };
+    }
+  } else {
+    for (const testcase of args.testcases) {
+      yield {
+        ...testcase,
+        testcaseId: 'id' in testcase ? testcase.id : null,
+        inputs: testcase.inputs as SystemInput,
+      };
+    }
+  }
+}
 
 /**
  * Runs a system on a Testset and records the results in Scorecard.
  *
  * @param scorecard The Scorecard client
- * @param projectId The ID of the Project to run the system on.
- * @param testsetId The ID of the Testset to run the system on.
- * @param metricIds The IDs of the Metrics to use for evaluation.
- * @param system The system to run on the Testset.
+ * @param args.projectId The ID of the Project to run the system on.
+ * @param args.testsetId The optional ID of the Testset to run the system on. Either this or `args.testcases` must be provided.
+ * @param args.testcases The optional list of Testcases to run the system on. Either this or `args.testsetId` must be provided.
+ * @param args.metricIds The IDs of the Metrics to use for evaluation.
+ * @param args.systemConfigId The optional ID of the System Configuration to associate with the Run.
+ * @param args.system The system to run on the Testset.
+ * @param options.runInParallel Whether to call `args.system` in parallel. False (sequential) by default.
  */
-export async function runAndEvaluate<SystemInput extends Object, SystemOutput extends Object>(
+export async function runAndEvaluate<
+  SystemInput extends Record<string, any>,
+  SystemOutput extends Record<string, any>,
+>(
   scorecard: Scorecard,
-  {
-    projectId,
-    testsetId,
-    metricIds,
-    system,
-  }: {
-    projectId: string;
-    testsetId: string;
-    metricIds: Array<string>;
-    system: (testcaseInput: SystemInput) => Promise<SystemOutput>;
+  args: RunAndEvaluateArgs<SystemInput, SystemOutput>,
+  options: {
+    runInParallel: boolean;
+  } = {
+    runInParallel: false,
   },
 ): Promise<Pick<Scorecard.Runs.Run, 'id'> & { url: string }> {
-  const run = await scorecard.runs.create(projectId, {
-    testsetId,
-    metricIds,
+  const hasSystemConfig = 'systemConfigId' in args;
+  const hasTestset = 'testsetId' in args;
+
+  const runPromise = scorecard.runs.create(args.projectId, {
+    testsetId: hasTestset ? args.testsetId : null,
+    metricIds: args.metricIds,
+    ...(hasSystemConfig ?
+      {
+        systemConfigId: args.systemConfigId,
+      }
+    : null),
   });
+  const systemConfig = hasSystemConfig ? await scorecard.systemConfigs.get(args.systemConfigId) : null;
+  const run = await runPromise;
+
+  const recordPromises: Array<Promise<unknown>> = [];
+
+  for await (const { testcaseId, inputs, expected } of testcaseIterator(scorecard, args)) {
+    const modelResponsePromise = hasSystemConfig ? args.system(inputs, systemConfig!) : args.system(inputs);
+
+    function createRecord(outputs: SystemOutput): Promise<unknown> {
+      return scorecard.records.create(run.id, {
+        inputs,
+        expected,
+        outputs,
+        ...(testcaseId != null ? { testcaseId } : null),
+      });
+    }
 
-  // Run each Testcase sequentially
-  const recordPromises: Array<Promise<any>> = [];
-  for await (const testcase of scorecard.testcases.list(run.testsetId)) {
-    const modelResponse = await system(testcase.inputs as SystemInput);
-    const promise = scorecard.records.create(run.id, {
-      testcaseId: testcase.id,
-      inputs: testcase.inputs,
-      expected: testcase.expected,
-      outputs: modelResponse as Record<string, unknown>,
-    });
-    recordPromises.push(promise);
+    if (options.runInParallel) {
+      recordPromises.push(modelResponsePromise.then(createRecord));
+    } else {
+      recordPromises.push(createRecord(await modelResponsePromise));
+    }
   }
   // Wait until all the Records are created
   await Promise.all(recordPromises);
 
-  const runUrl = `https://app.getscorecard.ai/projects/${projectId}/runs/grades/${run.id}`;
+  const runUrl = `${scorecard.baseAppURL}/projects/${args.projectId}/runs/${run.id}`;
 
   return { id: run.id, url: runUrl };
 }

package/src/resources/index.ts

@@ -16,7 +16,6 @@ export {
   type SystemConfig,
   type SystemConfigCreateParams,
   type SystemConfigListParams,
-  type SystemConfigGetParams,
   type SystemConfigsPaginatedResponse,
 } from './system-configs';
 export {

package/src/resources/runs.ts

@@ -13,8 +13,8 @@ export class Runs extends APIResource {
    * ```ts
    * const run = await client.runs.create('314', {
    *   metricIds: ['789', '101'],
-   *   testsetId: '246',
    *   systemConfigId: '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
+   *   testsetId: '246',
    * });
    * ```
    */
@@ -52,7 +52,7 @@ export interface Run {
   /**
    * The ID of the Testset this Run is testing.
    */
-  testsetId: string;
+  testsetId: string | null;
 
   /**
    * The ID of the system configuration this Run is using.
@@ -67,14 +67,14 @@ export interface RunCreateParams {
   metricIds: Array<string>;
 
   /**
-   * The ID of the Testset this Run is testing.
+   * The ID of the system configuration this Run is using.
    */
-  testsetId: string;
+  systemConfigId?: string;
 
   /**
-   * The ID of the system configuration this Run is using.
+   * The ID of the Testset this Run is testing.
    */
-  systemConfigId?: string;
+  testsetId?: string | null;
 }
 
 export declare namespace Runs {

package/src/resources/system-configs.ts

@@ -81,17 +81,11 @@ export class SystemConfigs extends APIResource {
    * ```ts
    * const systemConfig = await client.systemConfigs.get(
    *   '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
-   *   { systemId: '12345678-0a8b-4f66-b6f3-2ddcfa097257' },
    * );
    * ```
    */
-  get(
-    systemConfigID: string,
-    params: SystemConfigGetParams,
-    options?: RequestOptions,
-  ): APIPromise<SystemConfig> {
-    const { systemId } = params;
-    return this._client.get(path`/systems/${systemId}/configs/${systemConfigID}`, options);
+  get(systemConfigID: string, options?: RequestOptions): APIPromise<SystemConfig> {
+    return this._client.get(path`/systems/configs/${systemConfigID}`, options);
   }
 }
 
@@ -185,19 +179,11 @@ export namespace SystemConfigCreateParams {
 
 export interface SystemConfigListParams extends PaginatedResponseParams {}
 
-export interface SystemConfigGetParams {
-  /**
-   * The ID of the system the configuration belongs to.
-   */
-  systemId: string;
-}
-
 export declare namespace SystemConfigs {
   export {
     type SystemConfig as SystemConfig,
     type SystemConfigsPaginatedResponse as SystemConfigsPaginatedResponse,
     type SystemConfigCreateParams as SystemConfigCreateParams,
     type SystemConfigListParams as SystemConfigListParams,
-    type SystemConfigGetParams as SystemConfigGetParams,
   };
 }

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = '1.0.0-alpha.8'; // x-release-please-version
+export const VERSION = '1.0.0-alpha.9'; // x-release-please-version

package/version.d.mts

@@ -1,2 +1,2 @@
-export declare const VERSION = "1.0.0-alpha.8";
+export declare const VERSION = "1.0.0-alpha.9";
 //# sourceMappingURL=version.d.mts.map

package/version.d.ts

@@ -1,2 +1,2 @@
-export declare const VERSION = "1.0.0-alpha.8";
+export declare const VERSION = "1.0.0-alpha.9";
 //# sourceMappingURL=version.d.ts.map

package/version.js

@@ -1,5 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = '1.0.0-alpha.8'; // x-release-please-version
+exports.VERSION = '1.0.0-alpha.9'; // x-release-please-version
 //# sourceMappingURL=version.js.map

package/version.mjs

@@ -1,2 +1,2 @@
-export const VERSION = '1.0.0-alpha.8'; // x-release-please-version
+export const VERSION = '1.0.0-alpha.9'; // x-release-please-version
 //# sourceMappingURL=version.mjs.map

package/src/internal/shim-types.d.ts

@@ -1,28 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-/**
- * Shims for types that we can't always rely on being available globally.
- *
- * Note: these only exist at the type-level, there is no corresponding runtime
- * version for any of these symbols.
- */
-
-/**
- * In order to properly access the global `NodeJS` type, if it's available, we
- * need to make use of declaration shadowing. Without this, any checks for the
- * presence of `NodeJS.ReadableStream` will fail.
- */
-declare namespace NodeJS {
-  interface ReadableStream {}
-}
-
-type HasProperties<T> = keyof T extends never ? false : true;
-
-// @ts-ignore
-type _ReadableStream<R = any> =
-  // @ts-ignore
-  HasProperties<NodeJS.ReadableStream> extends true ? NodeJS.ReadableStream<R> : ReadableStream<R>;
-
-// @ts-ignore
-declare const _ReadableStream: unknown extends typeof ReadableStream ? never : typeof ReadableStream;
-export { _ReadableStream as ReadableStream };