scorecard-ai 1.0.0-alpha.9 → 1.0.0

package/lib/runAndEvaluate.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runAndEvaluate = runAndEvaluate;
+const error_1 = require("../error.js");
 /**
  * Returns an async generator over the given Testset or Testcases.
  */
@@ -8,18 +9,18 @@ async function* testcaseIterator(scorecard, args) {
     if ('testsetId' in args) {
         for await (const testcase of scorecard.testcases.list(args.testsetId)) {
             yield {
-                ...testcase,
                 testcaseId: testcase.id,
                 inputs: testcase.inputs,
+                expected: testcase.expected,
             };
         }
     }
     else {
         for (const testcase of args.testcases) {
             yield {
-                ...testcase,
                 testcaseId: 'id' in testcase ? testcase.id : null,
                 inputs: testcase.inputs,
+                expected: testcase.expected,
             };
         }
     }
@@ -32,42 +33,51 @@ async function* testcaseIterator(scorecard, args) {
  * @param args.testsetId The optional ID of the Testset to run the system on. Either this or `args.testcases` must be provided.
  * @param args.testcases The optional list of Testcases to run the system on. Either this or `args.testsetId` must be provided.
  * @param args.metricIds The IDs of the Metrics to use for evaluation.
- * @param args.systemConfigId The optional ID of the System Configuration to associate with the Run.
+ * @param args.systemVersionId The optional ID of the System Version to associate with the Run.
  * @param args.system The system to run on the Testset.
  * @param options.runInParallel Whether to call `args.system` in parallel. False (sequential) by default.
+ * @param options.trials The number of times to run the system on each Testcase. 1 by default.
  */
 async function runAndEvaluate(scorecard, args, options = {
     runInParallel: false,
+    trials: 1,
 }) {
-    const hasSystemConfig = 'systemConfigId' in args;
+    const runInParallel = options.runInParallel ?? false;
+    const trials = options.trials ?? 1;
+    if (!(Number.isInteger(trials) && trials >= 1)) {
+        throw new error_1.ScorecardError('trials must be a positive integer');
+    }
+    const hasSystemVersion = 'systemVersionId' in args;
     const hasTestset = 'testsetId' in args;
     const runPromise = scorecard.runs.create(args.projectId, {
         testsetId: hasTestset ? args.testsetId : null,
         metricIds: args.metricIds,
-        ...(hasSystemConfig ?
+        ...(hasSystemVersion ?
             {
-                systemConfigId: args.systemConfigId,
+                systemVersionId: args.systemVersionId,
             }
             : null),
     });
-    const systemConfig = hasSystemConfig ? await scorecard.systemConfigs.get(args.systemConfigId) : null;
+    const systemVersion = hasSystemVersion ? await scorecard.systems.versions.get(args.systemVersionId) : null;
     const run = await runPromise;
     const recordPromises = [];
     for await (const { testcaseId, inputs, expected } of testcaseIterator(scorecard, args)) {
-        const modelResponsePromise = hasSystemConfig ? args.system(inputs, systemConfig) : args.system(inputs);
-        function createRecord(outputs) {
-            return scorecard.records.create(run.id, {
-                inputs,
-                expected,
-                outputs,
-                ...(testcaseId != null ? { testcaseId } : null),
-            });
-        }
-        if (options.runInParallel) {
-            recordPromises.push(modelResponsePromise.then(createRecord));
-        }
-        else {
-            recordPromises.push(createRecord(await modelResponsePromise));
+        for (let i = 0; i < trials; i++) {
+            const modelResponsePromise = hasSystemVersion ? args.system(inputs, systemVersion) : args.system(inputs);
+            function createRecord(outputs) {
+                return scorecard.records.create(run.id, {
+                    inputs,
+                    expected,
+                    outputs,
+                    ...(testcaseId != null ? { testcaseId } : null),
+                });
+            }
+            if (runInParallel) {
+                recordPromises.push(modelResponsePromise.then(createRecord));
+            }
+            else {
+                recordPromises.push(createRecord(await modelResponsePromise));
+            }
         }
     }
     // Wait until all the Records are created

package/lib/runAndEvaluate.js.map

@@ -1 +1 @@
-{"version":3,"file":"runAndEvaluate.js","sourceRoot":"","sources":["../src/lib/runAndEvaluate.ts"],"names":[],"mappings":";;AAkGA,wCAqDC;AA/FD;;GAEG;AACH,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAC9B,SAAoB,EACpB,IAA0C;IAM1C,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACtE,MAAM;gBACJ,GAAG,QAAQ;gBACX,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,MAAM,EAAE,QAAQ,CAAC,MAAqB;aACvC,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,MAAM;gBACJ,GAAG,QAAQ;gBACX,UAAU,EAAE,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI;gBACjD,MAAM,EAAE,QAAQ,CAAC,MAAqB;aACvC,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;;;;;GAWG;AACI,KAAK,UAAU,cAAc,CAIlC,SAAoB,EACpB,IAAmD,EACnD,UAEI;IACF,aAAa,EAAE,KAAK;CACrB;IAED,MAAM,eAAe,GAAG,gBAAgB,IAAI,IAAI,CAAC;IACjD,MAAM,UAAU,GAAG,WAAW,IAAI,IAAI,CAAC;IAEvC,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE;QACvD,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QAC7C,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,GAAG,CAAC,eAAe,CAAC,CAAC;YACnB;gBACE,cAAc,EAAE,IAAI,CAAC,cAAc;aACpC;YACH,CAAC,CAAC,IAAI,CAAC;KACR,CAAC,CAAC;IACH,MAAM,YAAY,GAAG,eAAe,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACrG,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC;IAE7B,MAAM,cAAc,GAA4B,EAAE,CAAC;IAEnD,IAAI,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;QACvF,MAAM,oBAAoB,GAAG,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,YAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAExG,SAAS,YAAY,CAAC,OAAqB;YACzC,OAAO,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE;gBACtC,MAAM;gBACN,QAAQ;gBACR,OAAO;gBACP,GAAG,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aAChD,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YAC1B,cAAc,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QAC/D,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IACD,yCAAyC;IACzC,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAElC,MAAM,MAAM,GAAG,GAAG,SAAS,CAAC,UAAU,aAAa,IAAI,CAAC,SAAS,SAAS,GAAG,CAAC,EAAE,EAAE,CAAC;IAEnF,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AACrC,CAAC"}
+{"version":3,"file":"runAndEvaluate.js","sourceRoot":"","sources":["../src/lib/runAndEvaluate.ts"],"names":[],"mappings":";;AAqGA,wCA2EC;AA9KD,uCAA0C;AAwD1C;;GAEG;AACH,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAC9B,SAAoB,EACpB,IAA0C;IAM1C,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACtE,MAAM;gBACJ,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,MAAM,EAAE,QAAQ,CAAC,MAAqB;gBACtC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,MAAM;gBACJ,UAAU,EAAE,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI;gBACjD,MAAM,EAAE,QAAQ,CAAC,MAAqB;gBACtC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;GAYG;AACI,KAAK,UAAU,cAAc,CAIlC,SAAoB,EACpB,IAAmD,EACnD,UASI;IACF,aAAa,EAAE,KAAK;IACpB,MAAM,EAAE,CAAC;CACV;IAOD,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,KAAK,CAAC;IACrD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,sBAAc,CAAC,mCAAmC,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,gBAAgB,GAAG,iBAAiB,IAAI,IAAI,CAAC;IACnD,MAAM,UAAU,GAAG,WAAW,IAAI,IAAI,CAAC;IAEvC,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE;QACvD,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QAC7C,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,GAAG,CAAC,gBAAgB,CAAC,CAAC;YACpB;gBACE,eAAe,EAAE,IAAI,CAAC,eAAe;aACtC;YACH,CAAC,CAAC,IAAI,CAAC;KACR,CAAC,CAAC;IACH,MAAM,aAAa,GAAG,gBAAgB,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3G,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC;IAE7B,MAAM,cAAc,GAA4B,EAAE,CAAC;IAEnD,IAAI,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;QACvF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,oBAAoB,GACxB,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,aAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAE/E,SAAS,YAAY,CAAC,OAAqB;gBACzC,OAAO,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE;oBACtC,MAAM;oBACN,QAAQ;oBACR,OAAO;oBACP,GAAG,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAChD,CAAC,CAAC;YACL,CAAC;YAED,IAAI,aAAa,EAAE,CAAC;gBAClB,cAAc,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;YAC/D,CAAC;iBAAM,CAAC;gBACN,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;IACH,CAAC;IACD,yCAAyC;IACzC,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAElC,MAAM,MAAM,GAAG,GAAG,SAAS,CAAC,UAAU,aAAa,IAAI,CAAC,SAAS,SAAS,GAAG,CAAC,EAAE,EAAE,CAAC;IAEnF,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AACrC,CAAC"}

package/lib/runAndEvaluate.mjs

@@ -1,3 +1,4 @@
+import { ScorecardError } from "../error.mjs";
 /**
  * Returns an async generator over the given Testset or Testcases.
  */
@@ -5,18 +6,18 @@ async function* testcaseIterator(scorecard, args) {
     if ('testsetId' in args) {
         for await (const testcase of scorecard.testcases.list(args.testsetId)) {
             yield {
-                ...testcase,
                 testcaseId: testcase.id,
                 inputs: testcase.inputs,
+                expected: testcase.expected,
             };
         }
     }
     else {
         for (const testcase of args.testcases) {
             yield {
-                ...testcase,
                 testcaseId: 'id' in testcase ? testcase.id : null,
                 inputs: testcase.inputs,
+                expected: testcase.expected,
             };
         }
     }
@@ -29,42 +30,51 @@ async function* testcaseIterator(scorecard, args) {
  * @param args.testsetId The optional ID of the Testset to run the system on. Either this or `args.testcases` must be provided.
  * @param args.testcases The optional list of Testcases to run the system on. Either this or `args.testsetId` must be provided.
  * @param args.metricIds The IDs of the Metrics to use for evaluation.
- * @param args.systemConfigId The optional ID of the System Configuration to associate with the Run.
+ * @param args.systemVersionId The optional ID of the System Version to associate with the Run.
  * @param args.system The system to run on the Testset.
  * @param options.runInParallel Whether to call `args.system` in parallel. False (sequential) by default.
+ * @param options.trials The number of times to run the system on each Testcase. 1 by default.
  */
 export async function runAndEvaluate(scorecard, args, options = {
     runInParallel: false,
+    trials: 1,
 }) {
-    const hasSystemConfig = 'systemConfigId' in args;
+    const runInParallel = options.runInParallel ?? false;
+    const trials = options.trials ?? 1;
+    if (!(Number.isInteger(trials) && trials >= 1)) {
+        throw new ScorecardError('trials must be a positive integer');
+    }
+    const hasSystemVersion = 'systemVersionId' in args;
     const hasTestset = 'testsetId' in args;
     const runPromise = scorecard.runs.create(args.projectId, {
         testsetId: hasTestset ? args.testsetId : null,
         metricIds: args.metricIds,
-        ...(hasSystemConfig ?
+        ...(hasSystemVersion ?
             {
-                systemConfigId: args.systemConfigId,
+                systemVersionId: args.systemVersionId,
             }
             : null),
     });
-    const systemConfig = hasSystemConfig ? await scorecard.systemConfigs.get(args.systemConfigId) : null;
+    const systemVersion = hasSystemVersion ? await scorecard.systems.versions.get(args.systemVersionId) : null;
     const run = await runPromise;
     const recordPromises = [];
     for await (const { testcaseId, inputs, expected } of testcaseIterator(scorecard, args)) {
-        const modelResponsePromise = hasSystemConfig ? args.system(inputs, systemConfig) : args.system(inputs);
-        function createRecord(outputs) {
-            return scorecard.records.create(run.id, {
-                inputs,
-                expected,
-                outputs,
-                ...(testcaseId != null ? { testcaseId } : null),
-            });
-        }
-        if (options.runInParallel) {
-            recordPromises.push(modelResponsePromise.then(createRecord));
-        }
-        else {
-            recordPromises.push(createRecord(await modelResponsePromise));
+        for (let i = 0; i < trials; i++) {
+            const modelResponsePromise = hasSystemVersion ? args.system(inputs, systemVersion) : args.system(inputs);
+            function createRecord(outputs) {
+                return scorecard.records.create(run.id, {
+                    inputs,
+                    expected,
+                    outputs,
+                    ...(testcaseId != null ? { testcaseId } : null),
+                });
+            }
+            if (runInParallel) {
+                recordPromises.push(modelResponsePromise.then(createRecord));
+            }
+            else {
+                recordPromises.push(createRecord(await modelResponsePromise));
+            }
         }
     }
     // Wait until all the Records are created

package/lib/runAndEvaluate.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"runAndEvaluate.mjs","sourceRoot":"","sources":["../src/lib/runAndEvaluate.ts"],"names":[],"mappings":"AAwDA;;GAEG;AACH,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAC9B,SAAoB,EACpB,IAA0C;IAM1C,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACtE,MAAM;gBACJ,GAAG,QAAQ;gBACX,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,MAAM,EAAE,QAAQ,CAAC,MAAqB;aACvC,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,MAAM;gBACJ,GAAG,QAAQ;gBACX,UAAU,EAAE,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI;gBACjD,MAAM,EAAE,QAAQ,CAAC,MAAqB;aACvC,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAIlC,SAAoB,EACpB,IAAmD,EACnD,UAEI;IACF,aAAa,EAAE,KAAK;CACrB;IAED,MAAM,eAAe,GAAG,gBAAgB,IAAI,IAAI,CAAC;IACjD,MAAM,UAAU,GAAG,WAAW,IAAI,IAAI,CAAC;IAEvC,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE;QACvD,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QAC7C,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,GAAG,CAAC,eAAe,CAAC,CAAC;YACnB;gBACE,cAAc,EAAE,IAAI,CAAC,cAAc;aACpC;YACH,CAAC,CAAC,IAAI,CAAC;KACR,CAAC,CAAC;IACH,MAAM,YAAY,GAAG,eAAe,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACrG,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC;IAE7B,MAAM,cAAc,GAA4B,EAAE,CAAC;IAEnD,IAAI,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;QACvF,MAAM,oBAAoB,GAAG,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,YAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAExG,SAAS,YAAY,CAAC,OAAqB;YACzC,OAAO,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE;gBACtC,MAAM;gBACN,QAAQ;gBACR,OAAO;gBACP,GAAG,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aAChD,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YAC1B,cAAc,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QAC/D,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IACD,yCAAyC;IACzC,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAElC,MAAM,MAAM,GAAG,GAAG,SAAS,CAAC,UAAU,aAAa,IAAI,CAAC,SAAS,SAAS,GAAG,CAAC,EAAE,EAAE,CAAC;IAEnF,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AACrC,CAAC"}
+{"version":3,"file":"runAndEvaluate.mjs","sourceRoot":"","sources":["../src/lib/runAndEvaluate.ts"],"names":[],"mappings":"OAEO,EAAE,cAAc,EAAE;AAwDzB;;GAEG;AACH,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAC9B,SAAoB,EACpB,IAA0C;IAM1C,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACtE,MAAM;gBACJ,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,MAAM,EAAE,QAAQ,CAAC,MAAqB;gBACtC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,MAAM;gBACJ,UAAU,EAAE,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI;gBACjD,MAAM,EAAE,QAAQ,CAAC,MAAqB;gBACtC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAIlC,SAAoB,EACpB,IAAmD,EACnD,UASI;IACF,aAAa,EAAE,KAAK;IACpB,MAAM,EAAE,CAAC;CACV;IAOD,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,KAAK,CAAC;IACrD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,cAAc,CAAC,mCAAmC,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,gBAAgB,GAAG,iBAAiB,IAAI,IAAI,CAAC;IACnD,MAAM,UAAU,GAAG,WAAW,IAAI,IAAI,CAAC;IAEvC,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE;QACvD,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QAC7C,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,GAAG,CAAC,gBAAgB,CAAC,CAAC;YACpB;gBACE,eAAe,EAAE,IAAI,CAAC,eAAe;aACtC;YACH,CAAC,CAAC,IAAI,CAAC;KACR,CAAC,CAAC;IACH,MAAM,aAAa,GAAG,gBAAgB,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3G,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC;IAE7B,MAAM,cAAc,GAA4B,EAAE,CAAC;IAEnD,IAAI,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;QACvF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,oBAAoB,GACxB,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,aAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAE/E,SAAS,YAAY,CAAC,OAAqB;gBACzC,OAAO,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE;oBACtC,MAAM;oBACN,QAAQ;oBACR,OAAO;oBACP,GAAG,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAChD,CAAC,CAAC;YACL,CAAC;YAED,IAAI,aAAa,EAAE,CAAC;gBAClB,cAAc,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;YAC/D,CAAC;iBAAM,CAAC;gBACN,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;IACH,CAAC;IACD,yCAAyC;IACzC,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAElC,MAAM,MAAM,GAAG,GAAG,SAAS,CAAC,UAAU,aAAa,IAAI,CAAC,SAAS,SAAS,GAAG,CAAC,EAAE,EAAE,CAAC;IAEnF,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AACrC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scorecard-ai",
-  "version": "1.0.0-alpha.9",
+  "version": "1.0.0",
   "description": "The official TypeScript library for the Scorecard API",
   "author": "Scorecard <team@scorecard.io>",
   "types": "./index.d.ts",

package/resources/index.d.mts

@@ -1,10 +1,10 @@
 export * from "./shared.mjs";
+export { Metrics, type Metric, type MetricCreateParams } from "./metrics.mjs";
 export { Projects, type Project, type ProjectCreateParams, type ProjectListParams, type ProjectsPaginatedResponse, } from "./projects.mjs";
 export { Records, type Record, type RecordCreateParams } from "./records.mjs";
 export { Runs, type Run, type RunCreateParams } from "./runs.mjs";
 export { Scores, type Score, type ScoreUpsertParams } from "./scores.mjs";
-export { SystemConfigs, type SystemConfig, type SystemConfigCreateParams, type SystemConfigListParams, type SystemConfigsPaginatedResponse, } from "./system-configs.mjs";
-export { Systems, type System, type SystemDeleteResponse, type SystemCreateParams, type SystemUpdateParams, type SystemListParams, type SystemsPaginatedResponse, } from "./systems.mjs";
+export { Systems, type System, type SystemDeleteResponse, type SystemCreateParams, type SystemUpdateParams, type SystemListParams, type SystemsPaginatedResponse, } from "./systems/systems.mjs";
 export { Testcases, type Testcase, type TestcaseCreateResponse, type TestcaseDeleteResponse, type TestcaseCreateParams, type TestcaseUpdateParams, type TestcaseListParams, type TestcaseDeleteParams, type TestcasesPaginatedResponse, } from "./testcases.mjs";
 export { Testsets, type Testset, type TestsetDeleteResponse, type TestsetCreateParams, type TestsetUpdateParams, type TestsetListParams, type TestsetsPaginatedResponse, } from "./testsets.mjs";
 //# sourceMappingURL=index.d.mts.map

package/resources/index.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.mts","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":";OAGO,EACL,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC/B;OACM,EAAE,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,kBAAkB,EAAE;OACjD,EAAE,IAAI,EAAE,KAAK,GAAG,EAAE,KAAK,eAAe,EAAE;OACxC,EAAE,MAAM,EAAE,KAAK,KAAK,EAAE,KAAK,iBAAiB,EAAE;OAC9C,EACL,aAAa,EACb,KAAK,YAAY,EACjB,KAAK,wBAAwB,EAC7B,KAAK,sBAAsB,EAC3B,KAAK,8BAA8B,GACpC;OACM,EACL,OAAO,EACP,KAAK,MAAM,EACX,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,GAC9B;OACM,EACL,SAAS,EACT,KAAK,QAAQ,EACb,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAC3B,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,GAChC;OACM,EACL,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,qBAAqB,EAC1B,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC/B"}
+{"version":3,"file":"index.d.mts","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":";OAGO,EAAE,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,kBAAkB,EAAE;OACjD,EACL,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC/B;OACM,EAAE,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,kBAAkB,EAAE;OACjD,EAAE,IAAI,EAAE,KAAK,GAAG,EAAE,KAAK,eAAe,EAAE;OACxC,EAAE,MAAM,EAAE,KAAK,KAAK,EAAE,KAAK,iBAAiB,EAAE;OAC9C,EACL,OAAO,EACP,KAAK,MAAM,EACX,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,GAC9B;OACM,EACL,SAAS,EACT,KAAK,QAAQ,EACb,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAC3B,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,GAChC;OACM,EACL,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,qBAAqB,EAC1B,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC/B"}

package/resources/index.d.ts

@@ -1,10 +1,10 @@
 export * from "./shared.js";
+export { Metrics, type Metric, type MetricCreateParams } from "./metrics.js";
 export { Projects, type Project, type ProjectCreateParams, type ProjectListParams, type ProjectsPaginatedResponse, } from "./projects.js";
 export { Records, type Record, type RecordCreateParams } from "./records.js";
 export { Runs, type Run, type RunCreateParams } from "./runs.js";
 export { Scores, type Score, type ScoreUpsertParams } from "./scores.js";
-export { SystemConfigs, type SystemConfig, type SystemConfigCreateParams, type SystemConfigListParams, type SystemConfigsPaginatedResponse, } from "./system-configs.js";
-export { Systems, type System, type SystemDeleteResponse, type SystemCreateParams, type SystemUpdateParams, type SystemListParams, type SystemsPaginatedResponse, } from "./systems.js";
+export { Systems, type System, type SystemDeleteResponse, type SystemCreateParams, type SystemUpdateParams, type SystemListParams, type SystemsPaginatedResponse, } from "./systems/systems.js";
 export { Testcases, type Testcase, type TestcaseCreateResponse, type TestcaseDeleteResponse, type TestcaseCreateParams, type TestcaseUpdateParams, type TestcaseListParams, type TestcaseDeleteParams, type TestcasesPaginatedResponse, } from "./testcases.js";
 export { Testsets, type Testset, type TestsetDeleteResponse, type TestsetCreateParams, type TestsetUpdateParams, type TestsetListParams, type TestsetsPaginatedResponse, } from "./testsets.js";
 //# sourceMappingURL=index.d.ts.map

package/resources/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":";OAGO,EACL,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC/B;OACM,EAAE,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,kBAAkB,EAAE;OACjD,EAAE,IAAI,EAAE,KAAK,GAAG,EAAE,KAAK,eAAe,EAAE;OACxC,EAAE,MAAM,EAAE,KAAK,KAAK,EAAE,KAAK,iBAAiB,EAAE;OAC9C,EACL,aAAa,EACb,KAAK,YAAY,EACjB,KAAK,wBAAwB,EAC7B,KAAK,sBAAsB,EAC3B,KAAK,8BAA8B,GACpC;OACM,EACL,OAAO,EACP,KAAK,MAAM,EACX,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,GAC9B;OACM,EACL,SAAS,EACT,KAAK,QAAQ,EACb,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAC3B,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,GAChC;OACM,EACL,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,qBAAqB,EAC1B,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC/B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":";OAGO,EAAE,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,kBAAkB,EAAE;OACjD,EACL,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC/B;OACM,EAAE,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,kBAAkB,EAAE;OACjD,EAAE,IAAI,EAAE,KAAK,GAAG,EAAE,KAAK,eAAe,EAAE;OACxC,EAAE,MAAM,EAAE,KAAK,KAAK,EAAE,KAAK,iBAAiB,EAAE;OAC9C,EACL,OAAO,EACP,KAAK,MAAM,EACX,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,GAC9B;OACM,EACL,SAAS,EACT,KAAK,QAAQ,EACb,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAC3B,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,GAChC;OACM,EACL,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,qBAAqB,EAC1B,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC/B"}

package/resources/index.js

@@ -1,9 +1,11 @@
 "use strict";
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Testsets = exports.Testcases = exports.Systems = exports.SystemConfigs = exports.Scores = exports.Runs = exports.Records = exports.Projects = void 0;
+exports.Testsets = exports.Testcases = exports.Systems = exports.Scores = exports.Runs = exports.Records = exports.Projects = exports.Metrics = void 0;
 const tslib_1 = require("../internal/tslib.js");
 tslib_1.__exportStar(require("./shared.js"), exports);
+var metrics_1 = require("./metrics.js");
+Object.defineProperty(exports, "Metrics", { enumerable: true, get: function () { return metrics_1.Metrics; } });
 var projects_1 = require("./projects.js");
 Object.defineProperty(exports, "Projects", { enumerable: true, get: function () { return projects_1.Projects; } });
 var records_1 = require("./records.js");
@@ -12,9 +14,7 @@ var runs_1 = require("./runs.js");
 Object.defineProperty(exports, "Runs", { enumerable: true, get: function () { return runs_1.Runs; } });
 var scores_1 = require("./scores.js");
 Object.defineProperty(exports, "Scores", { enumerable: true, get: function () { return scores_1.Scores; } });
-var system_configs_1 = require("./system-configs.js");
-Object.defineProperty(exports, "SystemConfigs", { enumerable: true, get: function () { return system_configs_1.SystemConfigs; } });
-var systems_1 = require("./systems.js");
+var systems_1 = require("./systems/systems.js");
 Object.defineProperty(exports, "Systems", { enumerable: true, get: function () { return systems_1.Systems; } });
 var testcases_1 = require("./testcases.js");
 Object.defineProperty(exports, "Testcases", { enumerable: true, get: function () { return testcases_1.Testcases; } });

package/resources/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;;AAEtF,sDAAyB;AACzB,0CAMoB;AALlB,oGAAA,QAAQ,OAAA;AAMV,wCAA0E;AAAjE,kGAAA,OAAO,OAAA;AAChB,kCAA8D;AAArD,4FAAA,IAAI,OAAA;AACb,sCAAsE;AAA7D,gGAAA,MAAM,OAAA;AACf,sDAM0B;AALxB,+GAAA,aAAa,OAAA;AAMf,wCAQmB;AAPjB,kGAAA,OAAO,OAAA;AAQT,4CAUqB;AATnB,sGAAA,SAAS,OAAA;AAUX,0CAQoB;AAPlB,oGAAA,QAAQ,OAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;;AAEtF,sDAAyB;AACzB,wCAA0E;AAAjE,kGAAA,OAAO,OAAA;AAChB,0CAMoB;AALlB,oGAAA,QAAQ,OAAA;AAMV,wCAA0E;AAAjE,kGAAA,OAAO,OAAA;AAChB,kCAA8D;AAArD,4FAAA,IAAI,OAAA;AACb,sCAAsE;AAA7D,gGAAA,MAAM,OAAA;AACf,gDAQ2B;AAPzB,kGAAA,OAAO,OAAA;AAQT,4CAUqB;AATnB,sGAAA,SAAS,OAAA;AAUX,0CAQoB;AAPlB,oGAAA,QAAQ,OAAA"}

package/resources/index.mjs

@@ -1,11 +1,11 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 export * from "./shared.mjs";
+export { Metrics } from "./metrics.mjs";
 export { Projects, } from "./projects.mjs";
 export { Records } from "./records.mjs";
 export { Runs } from "./runs.mjs";
 export { Scores } from "./scores.mjs";
-export { SystemConfigs, } from "./system-configs.mjs";
-export { Systems, } from "./systems.mjs";
+export { Systems, } from "./systems/systems.mjs";
 export { Testcases, } from "./testcases.mjs";
 export { Testsets, } from "./testsets.mjs";
 //# sourceMappingURL=index.mjs.map

package/resources/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.mjs","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":"AAAA,sFAAsF;;OAG/E,EACL,QAAQ,GAKT;OACM,EAAE,OAAO,EAAwC;OACjD,EAAE,IAAI,EAAkC;OACxC,EAAE,MAAM,EAAsC;OAC9C,EACL,aAAa,GAKd;OACM,EACL,OAAO,GAOR;OACM,EACL,SAAS,GASV;OACM,EACL,QAAQ,GAOT"}
+{"version":3,"file":"index.mjs","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":"AAAA,sFAAsF;;OAG/E,EAAE,OAAO,EAAwC;OACjD,EACL,QAAQ,GAKT;OACM,EAAE,OAAO,EAAwC;OACjD,EAAE,IAAI,EAAkC;OACxC,EAAE,MAAM,EAAsC;OAC9C,EACL,OAAO,GAOR;OACM,EACL,SAAS,GASV;OACM,EACL,QAAQ,GAOT"}