scorecard-ai 1.0.0-alpha.9 → 1.0.0

package/resources/systems.mjs

@@ -1,139 +1,3 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-import { APIResource } from "../core/resource.mjs";
-import { PaginatedResponse } from "../core/pagination.mjs";
-import { path } from "../internal/utils/path.mjs";
-export class Systems extends APIResource {
-    /**
-     * Create a new system definition that specifies the interface contracts for a
-     * component you want to evaluate.
-     *
-     * A system acts as a template that defines three key contracts through JSON
-     * Schemas:
-     *
-     * 1. Input Schema: What data your system accepts (e.g., user queries, context
-     *    documents)
-     * 2. Output Schema: What data your system produces (e.g., responses, confidence
-     *    scores)
-     * 3. Config Schema: What parameters can be adjusted (e.g., model selection,
-     *    temperature)
-     *
-     * This separation lets you evaluate any system as a black box, focusing on its
-     * interface rather than implementation details.
-     *
-     * @example
-     * ```ts
-     * const system = await client.systems.create('314', {
-     *   configSchema: {
-     *     type: 'object',
-     *     properties: {
-     *       temperature: { type: 'number' },
-     *       maxTokens: { type: 'integer' },
-     *       model: { type: 'string', enum: ['gpt-4', 'gpt-4-turbo'] },
-     *     },
-     *     required: ['model'],
-     *   },
-     *   description: 'Production chatbot powered by GPT-4',
-     *   inputSchema: {
-     *     type: 'object',
-     *     properties: {
-     *       messages: {
-     *         type: 'array',
-     *         items: {
-     *           type: 'object',
-     *           properties: {
-     *             role: { type: 'string', enum: ['system', 'user', 'assistant'] },
-     *             content: { type: 'string' },
-     *           },
-     *           required: ['role', 'content'],
-     *         },
-     *       },
-     *     },
-     *     required: ['messages'],
-     *   },
-     *   name: 'GPT-4 Chatbot',
-     *   outputSchema: {
-     *     type: 'object',
-     *     properties: { response: { type: 'string' } },
-     *     required: ['response'],
-     *   },
-     * });
-     * ```
-     */
-    create(projectID, body, options) {
-        return this._client.post(path `/projects/${projectID}/systems`, { body, ...options });
-    }
-    /**
-     * Update an existing system definition. Only the fields provided in the request
-     * body will be updated. If a field is provided, the new content will replace the
-     * existing content. If a field is not provided, the existing content will remain
-     * unchanged.
-     *
-     * When updating schemas:
-     *
-     * - The system will accept your changes regardless of compatibility with existing
-     *   configurations
-     * - Schema updates won't invalidate existing evaluations or configurations
-     * - For significant redesigns, creating a new system definition provides a cleaner
-     *   separation
-     *
-     * @example
-     * ```ts
-     * const system = await client.systems.update(
-     *   '12345678-0a8b-4f66-b6f3-2ddcfa097257',
-     *   {
-     *     description:
-     *       'Updated production chatbot powered by GPT-4 Turbo',
-     *     name: 'GPT-4 Turbo Chatbot',
-     *   },
-     * );
-     * ```
-     */
-    update(systemID, body = {}, options) {
-        return this._client.patch(path `/systems/${systemID}`, { body, ...options });
-    }
-    /**
-     * Retrieve a paginated list of all systems. Systems are ordered by creation date.
-     *
-     * @example
-     * ```ts
-     * // Automatically fetches more pages as needed.
-     * for await (const system of client.systems.list('314')) {
-     *   // ...
-     * }
-     * ```
-     */
-    list(projectID, query = {}, options) {
-        return this._client.getAPIList(path `/projects/${projectID}/systems`, (PaginatedResponse), {
-            query,
-            ...options,
-        });
-    }
-    /**
-     * Delete a system definition by ID. This will not delete associated system
-     * configurations.
-     *
-     * @example
-     * ```ts
-     * const system = await client.systems.delete(
-     *   '12345678-0a8b-4f66-b6f3-2ddcfa097257',
-     * );
-     * ```
-     */
-    delete(systemID, options) {
-        return this._client.delete(path `/systems/${systemID}`, options);
-    }
-    /**
-     * Retrieve a specific system by ID.
-     *
-     * @example
-     * ```ts
-     * const system = await client.systems.get(
-     *   '12345678-0a8b-4f66-b6f3-2ddcfa097257',
-     * );
-     * ```
-     */
-    get(systemID, options) {
-        return this._client.get(path `/systems/${systemID}`, options);
-    }
-}
+export * from "./systems/index.mjs";
 //# sourceMappingURL=systems.mjs.map

package/resources/systems.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"systems.mjs","sourceRoot":"","sources":["../src/resources/systems.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;OAEf,EAAe,iBAAiB,EAAgC;OAEhE,EAAE,IAAI,EAAE;AAEf,MAAM,OAAO,OAAQ,SAAQ,WAAW;IACtC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuDG;IACH,MAAM,CAAC,SAAiB,EAAE,IAAwB,EAAE,OAAwB;QAC1E,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAA,aAAa,SAAS,UAAU,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACvF,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,MAAM,CACJ,QAAgB,EAChB,OAA8C,EAAE,EAChD,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAA,YAAY,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED;;;;;;;;;;OAUG;IACH,IAAI,CACF,SAAiB,EACjB,QAA6C,EAAE,EAC/C,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAA,aAAa,SAAS,UAAU,EAAE,CAAA,iBAAyB,CAAA,EAAE;YAC9F,KAAK;YACL,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,MAAM,CAAC,QAAgB,EAAE,OAAwB;QAC/C,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAA,YAAY,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;IAClE,CAAC;IAED;;;;;;;;;OASG;IACH,GAAG,CAAC,QAAgB,EAAE,OAAwB;QAC5C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,YAAY,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;IAC/D,CAAC;CACF"}
+{"version":3,"file":"systems.mjs","sourceRoot":"","sources":["../src/resources/systems.ts"],"names":[],"mappings":"AAAA,sFAAsF"}

package/resources/testsets.d.mts

@@ -75,7 +75,7 @@ export declare class Testsets extends APIResource {
      */
     delete(testsetID: string, options?: RequestOptions): APIPromise<TestsetDeleteResponse>;
     /**
-     * Get Testset by ID
+     * Get Testset
      *
      * @example
      * ```ts

package/resources/testsets.d.ts

@@ -75,7 +75,7 @@ export declare class Testsets extends APIResource {
      */
     delete(testsetID: string, options?: RequestOptions): APIPromise<TestsetDeleteResponse>;
     /**
-     * Get Testset by ID
+     * Get Testset
      *
      * @example
      * ```ts

package/resources/testsets.js

@@ -89,7 +89,7 @@ class Testsets extends resource_1.APIResource {
         return this._client.delete((0, path_1.path) `/testsets/${testsetID}`, options);
     }
     /**
-     * Get Testset by ID
+     * Get Testset
      *
      * @example
      * ```ts

package/resources/testsets.mjs

@@ -86,7 +86,7 @@ export class Testsets extends APIResource {
         return this._client.delete(path `/testsets/${testsetID}`, options);
     }
     /**
-     * Get Testset by ID
+     * Get Testset
      *
      * @example
      * ```ts

package/src/client.ts

@@ -22,6 +22,7 @@ import { APIPromise } from './core/api-promise';
 import { type Fetch } from './internal/builtin-types';
 import { HeadersLike, NullableHeaders, buildHeaders } from './internal/headers';
 import { FinalRequestOptions, RequestOptions } from './internal/request-options';
+import { Metric, MetricCreateParams, Metrics } from './resources/metrics';
 import {
   Project,
   ProjectCreateParams,
@@ -32,22 +33,6 @@ import {
 import { Record as RecordsAPIRecord, RecordCreateParams, Records } from './resources/records';
 import { Run, RunCreateParams, Runs } from './resources/runs';
 import { Score, ScoreUpsertParams, Scores } from './resources/scores';
-import {
-  SystemConfig,
-  SystemConfigCreateParams,
-  SystemConfigListParams,
-  SystemConfigs,
-  SystemConfigsPaginatedResponse,
-} from './resources/system-configs';
-import {
-  System,
-  SystemCreateParams,
-  SystemDeleteResponse,
-  SystemListParams,
-  SystemUpdateParams,
-  Systems,
-  SystemsPaginatedResponse,
-} from './resources/systems';
 import {
   Testcase,
   TestcaseCreateParams,
@@ -71,6 +56,15 @@ import {
 import { readEnv } from './internal/utils/env';
 import { formatRequestDetails, loggerFor } from './internal/utils/log';
 import { isEmptyObj } from './internal/utils/values';
+import {
+  System,
+  SystemCreateParams,
+  SystemDeleteResponse,
+  SystemListParams,
+  SystemUpdateParams,
+  Systems,
+  SystemsPaginatedResponse,
+} from './resources/systems/systems';
 
 const environments = {
   production: 'https://api2.scorecard.io/api/v2',
@@ -801,19 +795,19 @@ export class Scorecard {
   testsets: API.Testsets = new API.Testsets(this);
   testcases: API.Testcases = new API.Testcases(this);
   runs: API.Runs = new API.Runs(this);
+  metrics: API.Metrics = new API.Metrics(this);
   records: API.Records = new API.Records(this);
   scores: API.Scores = new API.Scores(this);
   systems: API.Systems = new API.Systems(this);
-  systemConfigs: API.SystemConfigs = new API.SystemConfigs(this);
 }
 Scorecard.Projects = Projects;
 Scorecard.Testsets = Testsets;
 Scorecard.Testcases = Testcases;
 Scorecard.Runs = Runs;
+Scorecard.Metrics = Metrics;
 Scorecard.Records = Records;
 Scorecard.Scores = Scores;
 Scorecard.Systems = Systems;
-Scorecard.SystemConfigs = SystemConfigs;
 export declare namespace Scorecard {
   export type RequestOptions = Opts.RequestOptions;
 
@@ -855,6 +849,8 @@ export declare namespace Scorecard {
 
   export { Runs as Runs, type Run as Run, type RunCreateParams as RunCreateParams };
 
+  export { Metrics as Metrics, type Metric as Metric, type MetricCreateParams as MetricCreateParams };
+
   export {
     Records as Records,
     type RecordsAPIRecord as Record,
@@ -873,13 +869,5 @@ export declare namespace Scorecard {
     type SystemListParams as SystemListParams,
   };
 
-  export {
-    SystemConfigs as SystemConfigs,
-    type SystemConfig as SystemConfig,
-    type SystemConfigsPaginatedResponse as SystemConfigsPaginatedResponse,
-    type SystemConfigCreateParams as SystemConfigCreateParams,
-    type SystemConfigListParams as SystemConfigListParams,
-  };
-
   export type APIError = API.APIError;
 }

package/src/lib/runAndEvaluate.ts

@@ -1,5 +1,7 @@
 import { Scorecard } from '../client';
-import { SystemConfig, Testcase } from '../resources';
+import { Testcase } from '../resources';
+import { ScorecardError } from '../error';
+import { SystemVersion } from '../resources/systems';
 
 type RunAndEvaluateArgs<SystemInput extends Record<string, any>, SystemOutput extends Record<string, any>> =
   // Project and metrics are always required
@@ -14,17 +16,17 @@ type RunAndEvaluateArgs<SystemInput extends Record<string, any>, SystemOutput ex
      */
     metricIds: Array<string>;
   } & (
-    | // If system config is provided, the system function receives a system config
+    | // If systemVersionId is provided, the system function receives a system version
     {
         /**
-         * The ID of the System Configuration to use for the run.
+         * The ID of the SystemVersion to use for the run.
          */
-        systemConfigId: string;
+        systemVersionId: string;
 
         /**
          * The system function to run on the Testset.
          */
-        system: (testcaseInput: SystemInput, systemConfig: SystemConfig) => Promise<SystemOutput>;
+        system: (testcaseInput: SystemInput, systemVersion: SystemVersion) => Promise<SystemOutput>;
       }
     // Otherwise, the system function receives only the testcase input
     | {
@@ -68,17 +70,17 @@ async function* testcaseIterator<SystemInput extends Record<string, any>>(
   if ('testsetId' in args) {
     for await (const testcase of scorecard.testcases.list(args.testsetId)) {
       yield {
-        ...testcase,
         testcaseId: testcase.id,
         inputs: testcase.inputs as SystemInput,
+        expected: testcase.expected,
       };
     }
   } else {
     for (const testcase of args.testcases) {
       yield {
-        ...testcase,
         testcaseId: 'id' in testcase ? testcase.id : null,
         inputs: testcase.inputs as SystemInput,
+        expected: testcase.expected,
       };
     }
   }
@@ -92,9 +94,10 @@ async function* testcaseIterator<SystemInput extends Record<string, any>>(
  * @param args.testsetId The optional ID of the Testset to run the system on. Either this or `args.testcases` must be provided.
  * @param args.testcases The optional list of Testcases to run the system on. Either this or `args.testsetId` must be provided.
  * @param args.metricIds The IDs of the Metrics to use for evaluation.
- * @param args.systemConfigId The optional ID of the System Configuration to associate with the Run.
+ * @param args.systemVersionId The optional ID of the System Version to associate with the Run.
  * @param args.system The system to run on the Testset.
  * @param options.runInParallel Whether to call `args.system` in parallel. False (sequential) by default.
+ * @param options.trials The number of times to run the system on each Testcase. 1 by default.
  */
 export async function runAndEvaluate<
   SystemInput extends Record<string, any>,
@@ -103,44 +106,66 @@ export async function runAndEvaluate<
   scorecard: Scorecard,
   args: RunAndEvaluateArgs<SystemInput, SystemOutput>,
   options: {
-    runInParallel: boolean;
+    /**
+     * Whether to call `args.system` in parallel. False (sequential) by default.
+     */
+    runInParallel?: boolean;
+    /**
+     * The number of times to run the system on each Testcase. 1 by default.
+     */
+    trials?: number;
   } = {
     runInParallel: false,
+    trials: 1,
   },
-): Promise<Pick<Scorecard.Runs.Run, 'id'> & { url: string }> {
-  const hasSystemConfig = 'systemConfigId' in args;
+): Promise<{
+  /** The ID of the Run. */
+  id: string;
+  /** The URL of the Run. */
+  url: string;
+}> {
+  const runInParallel = options.runInParallel ?? false;
+  const trials = options.trials ?? 1;
+  if (!(Number.isInteger(trials) && trials >= 1)) {
+    throw new ScorecardError('trials must be a positive integer');
+  }
+
+  const hasSystemVersion = 'systemVersionId' in args;
   const hasTestset = 'testsetId' in args;
 
   const runPromise = scorecard.runs.create(args.projectId, {
     testsetId: hasTestset ? args.testsetId : null,
     metricIds: args.metricIds,
-    ...(hasSystemConfig ?
+    ...(hasSystemVersion ?
       {
-        systemConfigId: args.systemConfigId,
+        systemVersionId: args.systemVersionId,
       }
     : null),
   });
-  const systemConfig = hasSystemConfig ? await scorecard.systemConfigs.get(args.systemConfigId) : null;
+  const systemVersion = hasSystemVersion ? await scorecard.systems.versions.get(args.systemVersionId) : null;
   const run = await runPromise;
 
   const recordPromises: Array<Promise<unknown>> = [];
 
   for await (const { testcaseId, inputs, expected } of testcaseIterator(scorecard, args)) {
-    const modelResponsePromise = hasSystemConfig ? args.system(inputs, systemConfig!) : args.system(inputs);
+    for (let i = 0; i < trials; i++) {
+      const modelResponsePromise =
+        hasSystemVersion ? args.system(inputs, systemVersion!) : args.system(inputs);
 
-    function createRecord(outputs: SystemOutput): Promise<unknown> {
-      return scorecard.records.create(run.id, {
-        inputs,
-        expected,
-        outputs,
-        ...(testcaseId != null ? { testcaseId } : null),
-      });
-    }
+      function createRecord(outputs: SystemOutput): Promise<unknown> {
+        return scorecard.records.create(run.id, {
+          inputs,
+          expected,
+          outputs,
+          ...(testcaseId != null ? { testcaseId } : null),
+        });
+      }
 
-    if (options.runInParallel) {
-      recordPromises.push(modelResponsePromise.then(createRecord));
-    } else {
-      recordPromises.push(createRecord(await modelResponsePromise));
+      if (runInParallel) {
+        recordPromises.push(modelResponsePromise.then(createRecord));
+      } else {
+        recordPromises.push(createRecord(await modelResponsePromise));
+      }
     }
   }
   // Wait until all the Records are created

package/src/resources/index.ts

@@ -1,6 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 export * from './shared';
+export { Metrics, type Metric, type MetricCreateParams } from './metrics';
 export {
   Projects,
   type Project,
@@ -11,13 +12,6 @@ export {
 export { Records, type Record, type RecordCreateParams } from './records';
 export { Runs, type Run, type RunCreateParams } from './runs';
 export { Scores, type Score, type ScoreUpsertParams } from './scores';
-export {
-  SystemConfigs,
-  type SystemConfig,
-  type SystemConfigCreateParams,
-  type SystemConfigListParams,
-  type SystemConfigsPaginatedResponse,
-} from './system-configs';
 export {
   Systems,
   type System,
@@ -26,7 +20,7 @@ export {
   type SystemUpdateParams,
   type SystemListParams,
   type SystemsPaginatedResponse,
-} from './systems';
+} from './systems/systems';
 export {
   Testcases,
   type Testcase,