scorecard-ai 1.0.0-alpha.8 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/CHANGELOG.md +46 -0
  2. package/README.md +25 -37
  3. package/client.d.mts +5 -4
  4. package/client.d.mts.map +1 -1
  5. package/client.d.ts +5 -4
  6. package/client.d.ts.map +1 -1
  7. package/client.js +19 -4
  8. package/client.js.map +1 -1
  9. package/client.mjs +19 -4
  10. package/client.mjs.map +1 -1
  11. package/core/pagination.d.mts +1 -1
  12. package/core/pagination.d.mts.map +1 -1
  13. package/core/pagination.d.ts +1 -1
  14. package/core/pagination.d.ts.map +1 -1
  15. package/index.d.mts +1 -0
  16. package/index.d.mts.map +1 -1
  17. package/index.d.ts +1 -0
  18. package/index.d.ts.map +1 -1
  19. package/index.js +3 -1
  20. package/index.js.map +1 -1
  21. package/index.mjs +1 -0
  22. package/index.mjs.map +1 -1
  23. package/internal/detect-platform.js +3 -3
  24. package/internal/detect-platform.js.map +1 -1
  25. package/internal/detect-platform.mjs +3 -3
  26. package/internal/detect-platform.mjs.map +1 -1
  27. package/internal/shim-types.d.mts +11 -22
  28. package/internal/shim-types.d.mts.map +1 -0
  29. package/internal/shim-types.d.ts +11 -22
  30. package/internal/shim-types.d.ts.map +1 -0
  31. package/internal/shim-types.js +4 -0
  32. package/internal/shim-types.js.map +1 -0
  33. package/internal/shim-types.mjs +3 -0
  34. package/internal/shim-types.mjs.map +1 -0
  35. package/internal/shims.d.mts +2 -2
  36. package/internal/shims.d.mts.map +1 -1
  37. package/internal/shims.d.ts +2 -2
  38. package/internal/shims.d.ts.map +1 -1
  39. package/internal/tslib.js +6 -6
  40. package/internal/uploads.js.map +1 -1
  41. package/internal/uploads.mjs.map +1 -1
  42. package/lib/runAndEvaluate.d.mts +62 -10
  43. package/lib/runAndEvaluate.d.mts.map +1 -1
  44. package/lib/runAndEvaluate.d.ts +62 -10
  45. package/lib/runAndEvaluate.d.ts.map +1 -1
  46. package/lib/runAndEvaluate.js +72 -19
  47. package/lib/runAndEvaluate.js.map +1 -1
  48. package/lib/runAndEvaluate.mjs +72 -19
  49. package/lib/runAndEvaluate.mjs.map +1 -1
  50. package/package.json +1 -4
  51. package/resources/index.d.mts +2 -2
  52. package/resources/index.d.mts.map +1 -1
  53. package/resources/index.d.ts +2 -2
  54. package/resources/index.d.ts.map +1 -1
  55. package/resources/index.js +4 -4
  56. package/resources/index.js.map +1 -1
  57. package/resources/index.mjs +2 -2
  58. package/resources/index.mjs.map +1 -1
  59. package/resources/metrics.d.mts +421 -0
  60. package/resources/metrics.d.mts.map +1 -0
  61. package/resources/metrics.d.ts +421 -0
  62. package/resources/metrics.d.ts.map +1 -0
  63. package/resources/metrics.js +33 -0
  64. package/resources/metrics.js.map +1 -0
  65. package/resources/metrics.mjs +29 -0
  66. package/resources/metrics.mjs.map +1 -0
  67. package/resources/runs.d.mts +8 -8
  68. package/resources/runs.d.mts.map +1 -1
  69. package/resources/runs.d.ts +8 -8
  70. package/resources/runs.d.ts.map +1 -1
  71. package/resources/runs.js +1 -1
  72. package/resources/runs.mjs +1 -1
  73. package/resources/systems/index.d.mts +3 -0
  74. package/resources/systems/index.d.mts.map +1 -0
  75. package/resources/systems/index.d.ts +3 -0
  76. package/resources/systems/index.d.ts.map +1 -0
  77. package/resources/systems/index.js +9 -0
  78. package/resources/systems/index.js.map +1 -0
  79. package/resources/systems/index.mjs +4 -0
  80. package/resources/systems/index.mjs.map +1 -0
  81. package/resources/systems/systems.d.mts +229 -0
  82. package/resources/systems/systems.d.mts.map +1 -0
  83. package/resources/systems/systems.d.ts +229 -0
  84. package/resources/systems/systems.d.ts.map +1 -0
  85. package/resources/systems/systems.js +151 -0
  86. package/resources/systems/systems.js.map +1 -0
  87. package/resources/systems/systems.mjs +146 -0
  88. package/resources/systems/systems.mjs.map +1 -0
  89. package/resources/systems/versions.d.mts +132 -0
  90. package/resources/systems/versions.d.mts.map +1 -0
  91. package/resources/systems/versions.d.ts +132 -0
  92. package/resources/systems/versions.d.ts.map +1 -0
  93. package/resources/systems/versions.js +82 -0
  94. package/resources/systems/versions.js.map +1 -0
  95. package/resources/systems/versions.mjs +78 -0
  96. package/resources/systems/versions.mjs.map +1 -0
  97. package/resources/systems.d.mts +1 -224
  98. package/resources/systems.d.mts.map +1 -1
  99. package/resources/systems.d.ts +1 -224
  100. package/resources/systems.d.ts.map +1 -1
  101. package/resources/systems.js +2 -139
  102. package/resources/systems.js.map +1 -1
  103. package/resources/systems.mjs +1 -137
  104. package/resources/systems.mjs.map +1 -1
  105. package/resources/testsets.d.mts +1 -1
  106. package/resources/testsets.d.ts +1 -1
  107. package/resources/testsets.js +1 -1
  108. package/resources/testsets.mjs +1 -1
  109. package/src/client.ts +28 -28
  110. package/src/core/pagination.ts +1 -1
  111. package/src/index.ts +2 -0
  112. package/src/internal/detect-platform.ts +3 -3
  113. package/src/internal/shim-types.ts +26 -0
  114. package/src/internal/shims.ts +2 -2
  115. package/src/internal/uploads.ts +1 -1
  116. package/src/lib/runAndEvaluate.ts +159 -31
  117. package/src/resources/index.ts +2 -9
  118. package/src/resources/metrics.ts +525 -0
  119. package/src/resources/runs.ts +8 -8
  120. package/src/resources/systems/index.ts +18 -0
  121. package/src/resources/systems/systems.ts +299 -0
  122. package/src/resources/systems/versions.ts +166 -0
  123. package/src/resources/systems.ts +1 -277
  124. package/src/resources/testsets.ts +1 -1
  125. package/src/version.ts +1 -1
  126. package/version.d.mts +1 -1
  127. package/version.d.mts.map +1 -1
  128. package/version.d.ts +1 -1
  129. package/version.d.ts.map +1 -1
  130. package/version.js +1 -1
  131. package/version.js.map +1 -1
  132. package/version.mjs +1 -1
  133. package/version.mjs.map +1 -1
  134. package/resources/system-configs.d.mts +0 -155
  135. package/resources/system-configs.d.mts.map +0 -1
  136. package/resources/system-configs.d.ts +0 -155
  137. package/resources/system-configs.d.ts.map +0 -1
  138. package/resources/system-configs.js +0 -83
  139. package/resources/system-configs.js.map +0 -1
  140. package/resources/system-configs.mjs +0 -79
  141. package/resources/system-configs.mjs.map +0 -1
  142. package/src/internal/shim-types.d.ts +0 -28
  143. package/src/resources/system-configs.ts +0 -203
@@ -1,19 +1,71 @@
1
1
  import { Scorecard } from "../client.js";
2
+ import { Testcase } from "../resources.js";
3
+ import { SystemVersion } from "../resources/systems.js";
4
+ type RunAndEvaluateArgs<SystemInput extends Record<string, any>, SystemOutput extends Record<string, any>> = {
5
+ /**
6
+ * The ID of the Project to run the system on.
7
+ */
8
+ projectId: string;
9
+ /**
10
+ * The IDs of the Metrics to use for evaluation.
11
+ */
12
+ metricIds: Array<string>;
13
+ } & (// If systemVersionId is provided, the system function receives a system version
14
+ {
15
+ /**
16
+ * The ID of the SystemVersion to use for the run.
17
+ */
18
+ systemVersionId: string;
19
+ /**
20
+ * The system function to run on the Testset.
21
+ */
22
+ system: (testcaseInput: SystemInput, systemVersion: SystemVersion) => Promise<SystemOutput>;
23
+ } | {
24
+ /**
25
+ * The system function to run on the Testset.
26
+ */
27
+ system: (testcaseInput: SystemInput) => Promise<SystemOutput>;
28
+ }) & ({
29
+ /**
30
+ * The ID of the Scorecard Testset to run the system on.
31
+ */
32
+ testsetId: string;
33
+ } | {
34
+ /**
35
+ * The list of test cases to run the system on. Can be a list of Scorecard Testcases or a list of inputs and expected outputs.
36
+ */
37
+ testcases: Array<{
38
+ inputs: SystemInput;
39
+ expected: Record<string, unknown>;
40
+ }> | Array<Testcase>;
41
+ });
2
42
  /**
3
43
  * Runs a system on a Testset and records the results in Scorecard.
4
44
  *
5
45
  * @param scorecard The Scorecard client
6
- * @param projectId The ID of the Project to run the system on.
7
- * @param testsetId The ID of the Testset to run the system on.
8
- * @param metricIds The IDs of the Metrics to use for evaluation.
9
- * @param system The system to run on the Testset.
46
+ * @param args.projectId The ID of the Project to run the system on.
47
+ * @param args.testsetId The optional ID of the Testset to run the system on. Either this or `args.testcases` must be provided.
48
+ * @param args.testcases The optional list of Testcases to run the system on. Either this or `args.testsetId` must be provided.
49
+ * @param args.metricIds The IDs of the Metrics to use for evaluation.
50
+ * @param args.systemVersionId The optional ID of the System Version to associate with the Run.
51
+ * @param args.system The system to run on the Testset.
52
+ * @param options.runInParallel Whether to call `args.system` in parallel. False (sequential) by default.
53
+ * @param options.trials The number of times to run the system on each Testcase. 1 by default.
10
54
  */
11
- export declare function runAndEvaluate<SystemInput extends Object, SystemOutput extends Object>(scorecard: Scorecard, { projectId, testsetId, metricIds, system, }: {
12
- projectId: string;
13
- testsetId: string;
14
- metricIds: Array<string>;
15
- system: (testcaseInput: SystemInput) => Promise<SystemOutput>;
16
- }): Promise<Pick<Scorecard.Runs.Run, 'id'> & {
55
+ export declare function runAndEvaluate<SystemInput extends Record<string, any>, SystemOutput extends Record<string, any>>(scorecard: Scorecard, args: RunAndEvaluateArgs<SystemInput, SystemOutput>, options?: {
56
+ /**
57
+ * Whether to call `args.system` in parallel. False (sequential) by default.
58
+ */
59
+ runInParallel?: boolean;
60
+ /**
61
+ * The number of times to run the system on each Testcase. 1 by default.
62
+ */
63
+ trials?: number;
64
+ }): Promise<{
65
+ /** The ID of the Run. */
66
+ id: string;
67
+ /** The URL of the Run. */
17
68
  url: string;
18
69
  }>;
70
+ export {};
19
71
  //# sourceMappingURL=runAndEvaluate.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"runAndEvaluate.d.ts","sourceRoot":"","sources":["../src/lib/runAndEvaluate.ts"],"names":[],"mappings":"OAAO,EAAE,SAAS,EAAE;AAEpB;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAAC,WAAW,SAAS,MAAM,EAAE,YAAY,SAAS,MAAM,EAC1F,SAAS,EAAE,SAAS,EACpB,EACE,SAAS,EACT,SAAS,EACT,SAAS,EACT,MAAM,GACP,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACzB,MAAM,EAAE,CAAC,aAAa,EAAE,WAAW,KAAK,OAAO,CAAC,YAAY,CAAC,CAAC;CAC/D,GACA,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,CAwB3D"}
1
+ {"version":3,"file":"runAndEvaluate.d.ts","sourceRoot":"","sources":["../src/lib/runAndEvaluate.ts"],"names":[],"mappings":"OAAO,EAAE,SAAS,EAAE;OACb,EAAE,QAAQ,EAAE;OAEZ,EAAE,aAAa,EAAE;AAExB,KAAK,kBAAkB,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,YAAY,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,IAEvG;IACE;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CAC1B,GAAG,CACA,gFAAgF;AAClF;IACI;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,MAAM,EAAE,CAAC,aAAa,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,KAAK,OAAO,CAAC,YAAY,CAAC,CAAC;CAC7F,GAED;IACE;;OAEG;IACH,MAAM,EAAE,CAAC,aAAa,EAAE,WAAW,KAAK,OAAO,CAAC,YAAY,CAAC,CAAC;CAC/D,CACJ,GAEC,CAAG;IACG;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB,GACD;IACE;;OAEG;IACH,SAAS,EACL,KAAK,CAAC;QACJ,MAAM,EAAE,WAAW,CAAC;QACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC,CAAC,GACF,KAAK,CAAC,QAAQ,CAAC,CAAC;CACrB,CACJ,CAAC;AAgCN;;;;;;;;;;;;GAYG;AACH,wBAAsB,cAAc,CAClC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACvC,YAAY,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAExC,SAAS,EAAE,SAAS,EACpB,IAAI,EAAE,kBAAkB,CAAC,WAAW,EAAE,YAAY,CAAC,EACnD,OAAO,GAAE;IACP;;OAEG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;CAIjB,GACA,OAAO,CAAC;IACT,yBAAyB;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,0BAA0B;IAC1B,GAAG,EAAE,MAAM,CAAC;CACb,CAAC,CAmDD"}
@@ -1,35 +1,88 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.runAndEvaluate = runAndEvaluate;
4
+ const error_1 = require("../error.js");
5
+ /**
6
+ * Returns an async generator over the given Testset or Testcases.
7
+ */
8
+ async function* testcaseIterator(scorecard, args) {
9
+ if ('testsetId' in args) {
10
+ for await (const testcase of scorecard.testcases.list(args.testsetId)) {
11
+ yield {
12
+ testcaseId: testcase.id,
13
+ inputs: testcase.inputs,
14
+ expected: testcase.expected,
15
+ };
16
+ }
17
+ }
18
+ else {
19
+ for (const testcase of args.testcases) {
20
+ yield {
21
+ testcaseId: 'id' in testcase ? testcase.id : null,
22
+ inputs: testcase.inputs,
23
+ expected: testcase.expected,
24
+ };
25
+ }
26
+ }
27
+ }
4
28
  /**
5
29
  * Runs a system on a Testset and records the results in Scorecard.
6
30
  *
7
31
  * @param scorecard The Scorecard client
8
- * @param projectId The ID of the Project to run the system on.
9
- * @param testsetId The ID of the Testset to run the system on.
10
- * @param metricIds The IDs of the Metrics to use for evaluation.
11
- * @param system The system to run on the Testset.
32
+ * @param args.projectId The ID of the Project to run the system on.
33
+ * @param args.testsetId The optional ID of the Testset to run the system on. Either this or `args.testcases` must be provided.
34
+ * @param args.testcases The optional list of Testcases to run the system on. Either this or `args.testsetId` must be provided.
35
+ * @param args.metricIds The IDs of the Metrics to use for evaluation.
36
+ * @param args.systemVersionId The optional ID of the System Version to associate with the Run.
37
+ * @param args.system The system to run on the Testset.
38
+ * @param options.runInParallel Whether to call `args.system` in parallel. False (sequential) by default.
39
+ * @param options.trials The number of times to run the system on each Testcase. 1 by default.
12
40
  */
13
- async function runAndEvaluate(scorecard, { projectId, testsetId, metricIds, system, }) {
14
- const run = await scorecard.runs.create(projectId, {
15
- testsetId,
16
- metricIds,
41
+ async function runAndEvaluate(scorecard, args, options = {
42
+ runInParallel: false,
43
+ trials: 1,
44
+ }) {
45
+ const runInParallel = options.runInParallel ?? false;
46
+ const trials = options.trials ?? 1;
47
+ if (!(Number.isInteger(trials) && trials >= 1)) {
48
+ throw new error_1.ScorecardError('trials must be a positive integer');
49
+ }
50
+ const hasSystemVersion = 'systemVersionId' in args;
51
+ const hasTestset = 'testsetId' in args;
52
+ const runPromise = scorecard.runs.create(args.projectId, {
53
+ testsetId: hasTestset ? args.testsetId : null,
54
+ metricIds: args.metricIds,
55
+ ...(hasSystemVersion ?
56
+ {
57
+ systemVersionId: args.systemVersionId,
58
+ }
59
+ : null),
17
60
  });
18
- // Run each Testcase sequentially
61
+ const systemVersion = hasSystemVersion ? await scorecard.systems.versions.get(args.systemVersionId) : null;
62
+ const run = await runPromise;
19
63
  const recordPromises = [];
20
- for await (const testcase of scorecard.testcases.list(run.testsetId)) {
21
- const modelResponse = await system(testcase.inputs);
22
- const promise = scorecard.records.create(run.id, {
23
- testcaseId: testcase.id,
24
- inputs: testcase.inputs,
25
- expected: testcase.expected,
26
- outputs: modelResponse,
27
- });
28
- recordPromises.push(promise);
64
+ for await (const { testcaseId, inputs, expected } of testcaseIterator(scorecard, args)) {
65
+ for (let i = 0; i < trials; i++) {
66
+ const modelResponsePromise = hasSystemVersion ? args.system(inputs, systemVersion) : args.system(inputs);
67
+ function createRecord(outputs) {
68
+ return scorecard.records.create(run.id, {
69
+ inputs,
70
+ expected,
71
+ outputs,
72
+ ...(testcaseId != null ? { testcaseId } : null),
73
+ });
74
+ }
75
+ if (runInParallel) {
76
+ recordPromises.push(modelResponsePromise.then(createRecord));
77
+ }
78
+ else {
79
+ recordPromises.push(createRecord(await modelResponsePromise));
80
+ }
81
+ }
29
82
  }
30
83
  // Wait until all the Records are created
31
84
  await Promise.all(recordPromises);
32
- const runUrl = `https://app.getscorecard.ai/projects/${projectId}/runs/grades/${run.id}`;
85
+ const runUrl = `${scorecard.baseAppURL}/projects/${args.projectId}/runs/${run.id}`;
33
86
  return { id: run.id, url: runUrl };
34
87
  }
35
88
  //# sourceMappingURL=runAndEvaluate.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"runAndEvaluate.js","sourceRoot":"","sources":["../src/lib/runAndEvaluate.ts"],"names":[],"mappings":";;AAWA,wCAqCC;AA9CD;;;;;;;;GAQG;AACI,KAAK,UAAU,cAAc,CAClC,SAAoB,EACpB,EACE,SAAS,EACT,SAAS,EACT,SAAS,EACT,MAAM,GAMP;IAED,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;QACjD,SAAS;QACT,SAAS;KACV,CAAC,CAAC;IAEH,iCAAiC;IACjC,MAAM,cAAc,GAAwB,EAAE,CAAC;IAC/C,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QACrE,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAqB,CAAC,CAAC;QACnE,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE;YAC/C,UAAU,EAAE,QAAQ,CAAC,EAAE;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,OAAO,EAAE,aAAwC;SAClD,CAAC,CAAC;QACH,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IACD,yCAAyC;IACzC,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAElC,MAAM,MAAM,GAAG,wCAAwC,SAAS,gBAAgB,GAAG,CAAC,EAAE,EAAE,CAAC;IAEzF,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AACrC,CAAC"}
1
+ {"version":3,"file":"runAndEvaluate.js","sourceRoot":"","sources":["../src/lib/runAndEvaluate.ts"],"names":[],"mappings":";;AAqGA,wCA2EC;AA9KD,uCAA0C;AAwD1C;;GAEG;AACH,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAC9B,SAAoB,EACpB,IAA0C;IAM1C,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACtE,MAAM;gBACJ,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,MAAM,EAAE,QAAQ,CAAC,MAAqB;gBACtC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,MAAM;gBACJ,UAAU,EAAE,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI;gBACjD,MAAM,EAAE,QAAQ,CAAC,MAAqB;gBACtC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;GAYG;AACI,KAAK,UAAU,cAAc,CAIlC,SAAoB,EACpB,IAAmD,EACnD,UASI;IACF,aAAa,EAAE,KAAK;IACpB,MAAM,EAAE,CAAC;CACV;IAOD,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,KAAK,CAAC;IACrD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,sBAAc,CAAC,mCAAmC,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,gBAAgB,GAAG,iBAAiB,IAAI,IAAI,CAAC;IACnD,MAAM,UAAU,GAAG,WAAW,IAAI,IAAI,CAAC;IAEvC,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE;QACvD,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QAC7C,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,GAAG,CAAC,gBAAgB,CAAC,CAAC;YACpB;gBACE,eAAe,EAAE,IAAI,CAAC,eAAe;aACtC;YACH,CAAC,CAAC,IAAI,CAAC;KACR,CAAC,CAAC;IACH,MAAM,aAAa,GAAG,gBAAgB,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3G,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC;IAE7B,MAAM,cAAc,GAA4B,EAAE,CAAC;IAEnD,IAAI,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;QACvF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,oBAAoB,GACxB,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,aAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAE/E,SAAS,YAAY,CAAC,OAAqB;gBACzC,OAAO,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE;oBACtC,MAAM;oBACN,QAAQ;oBACR,OAAO;oBACP,GAAG,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAChD,CAAC,CAAC;YACL,CAAC;YAED,IAAI,aAAa,EAAE,CAAC;gBAClB,cAAc,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;YAC/D,CAAC;iBAAM,CAAC;gBACN,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;IACH,CAAC;IACD,yCAAyC;IACzC,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAElC,MAAM,MAAM,GAAG,GAAG,SAAS,CAAC,UAAU,aAAa,IAAI,CAAC,SAAS,SAAS,GAAG,CAAC,EAAE,EAAE,CAAC;IAEnF,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AACrC,CAAC"}
@@ -1,32 +1,85 @@
1
+ import { ScorecardError } from "../error.mjs";
2
+ /**
3
+ * Returns an async generator over the given Testset or Testcases.
4
+ */
5
+ async function* testcaseIterator(scorecard, args) {
6
+ if ('testsetId' in args) {
7
+ for await (const testcase of scorecard.testcases.list(args.testsetId)) {
8
+ yield {
9
+ testcaseId: testcase.id,
10
+ inputs: testcase.inputs,
11
+ expected: testcase.expected,
12
+ };
13
+ }
14
+ }
15
+ else {
16
+ for (const testcase of args.testcases) {
17
+ yield {
18
+ testcaseId: 'id' in testcase ? testcase.id : null,
19
+ inputs: testcase.inputs,
20
+ expected: testcase.expected,
21
+ };
22
+ }
23
+ }
24
+ }
1
25
  /**
2
26
  * Runs a system on a Testset and records the results in Scorecard.
3
27
  *
4
28
  * @param scorecard The Scorecard client
5
- * @param projectId The ID of the Project to run the system on.
6
- * @param testsetId The ID of the Testset to run the system on.
7
- * @param metricIds The IDs of the Metrics to use for evaluation.
8
- * @param system The system to run on the Testset.
29
+ * @param args.projectId The ID of the Project to run the system on.
30
+ * @param args.testsetId The optional ID of the Testset to run the system on. Either this or `args.testcases` must be provided.
31
+ * @param args.testcases The optional list of Testcases to run the system on. Either this or `args.testsetId` must be provided.
32
+ * @param args.metricIds The IDs of the Metrics to use for evaluation.
33
+ * @param args.systemVersionId The optional ID of the System Version to associate with the Run.
34
+ * @param args.system The system to run on the Testset.
35
+ * @param options.runInParallel Whether to call `args.system` in parallel. False (sequential) by default.
36
+ * @param options.trials The number of times to run the system on each Testcase. 1 by default.
9
37
  */
10
- export async function runAndEvaluate(scorecard, { projectId, testsetId, metricIds, system, }) {
11
- const run = await scorecard.runs.create(projectId, {
12
- testsetId,
13
- metricIds,
38
+ export async function runAndEvaluate(scorecard, args, options = {
39
+ runInParallel: false,
40
+ trials: 1,
41
+ }) {
42
+ const runInParallel = options.runInParallel ?? false;
43
+ const trials = options.trials ?? 1;
44
+ if (!(Number.isInteger(trials) && trials >= 1)) {
45
+ throw new ScorecardError('trials must be a positive integer');
46
+ }
47
+ const hasSystemVersion = 'systemVersionId' in args;
48
+ const hasTestset = 'testsetId' in args;
49
+ const runPromise = scorecard.runs.create(args.projectId, {
50
+ testsetId: hasTestset ? args.testsetId : null,
51
+ metricIds: args.metricIds,
52
+ ...(hasSystemVersion ?
53
+ {
54
+ systemVersionId: args.systemVersionId,
55
+ }
56
+ : null),
14
57
  });
15
- // Run each Testcase sequentially
58
+ const systemVersion = hasSystemVersion ? await scorecard.systems.versions.get(args.systemVersionId) : null;
59
+ const run = await runPromise;
16
60
  const recordPromises = [];
17
- for await (const testcase of scorecard.testcases.list(run.testsetId)) {
18
- const modelResponse = await system(testcase.inputs);
19
- const promise = scorecard.records.create(run.id, {
20
- testcaseId: testcase.id,
21
- inputs: testcase.inputs,
22
- expected: testcase.expected,
23
- outputs: modelResponse,
24
- });
25
- recordPromises.push(promise);
61
+ for await (const { testcaseId, inputs, expected } of testcaseIterator(scorecard, args)) {
62
+ for (let i = 0; i < trials; i++) {
63
+ const modelResponsePromise = hasSystemVersion ? args.system(inputs, systemVersion) : args.system(inputs);
64
+ function createRecord(outputs) {
65
+ return scorecard.records.create(run.id, {
66
+ inputs,
67
+ expected,
68
+ outputs,
69
+ ...(testcaseId != null ? { testcaseId } : null),
70
+ });
71
+ }
72
+ if (runInParallel) {
73
+ recordPromises.push(modelResponsePromise.then(createRecord));
74
+ }
75
+ else {
76
+ recordPromises.push(createRecord(await modelResponsePromise));
77
+ }
78
+ }
26
79
  }
27
80
  // Wait until all the Records are created
28
81
  await Promise.all(recordPromises);
29
- const runUrl = `https://app.getscorecard.ai/projects/${projectId}/runs/grades/${run.id}`;
82
+ const runUrl = `${scorecard.baseAppURL}/projects/${args.projectId}/runs/${run.id}`;
30
83
  return { id: run.id, url: runUrl };
31
84
  }
32
85
  //# sourceMappingURL=runAndEvaluate.mjs.map
@@ -1 +1 @@
1
- {"version":3,"file":"runAndEvaluate.mjs","sourceRoot":"","sources":["../src/lib/runAndEvaluate.ts"],"names":[],"mappings":"AAEA;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAoB,EACpB,EACE,SAAS,EACT,SAAS,EACT,SAAS,EACT,MAAM,GAMP;IAED,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;QACjD,SAAS;QACT,SAAS;KACV,CAAC,CAAC;IAEH,iCAAiC;IACjC,MAAM,cAAc,GAAwB,EAAE,CAAC;IAC/C,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QACrE,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAqB,CAAC,CAAC;QACnE,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE;YAC/C,UAAU,EAAE,QAAQ,CAAC,EAAE;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,OAAO,EAAE,aAAwC;SAClD,CAAC,CAAC;QACH,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IACD,yCAAyC;IACzC,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAElC,MAAM,MAAM,GAAG,wCAAwC,SAAS,gBAAgB,GAAG,CAAC,EAAE,EAAE,CAAC;IAEzF,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AACrC,CAAC"}
1
+ {"version":3,"file":"runAndEvaluate.mjs","sourceRoot":"","sources":["../src/lib/runAndEvaluate.ts"],"names":[],"mappings":"OAEO,EAAE,cAAc,EAAE;AAwDzB;;GAEG;AACH,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAC9B,SAAoB,EACpB,IAA0C;IAM1C,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACtE,MAAM;gBACJ,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,MAAM,EAAE,QAAQ,CAAC,MAAqB;gBACtC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,MAAM;gBACJ,UAAU,EAAE,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI;gBACjD,MAAM,EAAE,QAAQ,CAAC,MAAqB;gBACtC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAIlC,SAAoB,EACpB,IAAmD,EACnD,UASI;IACF,aAAa,EAAE,KAAK;IACpB,MAAM,EAAE,CAAC;CACV;IAOD,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,KAAK,CAAC;IACrD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,cAAc,CAAC,mCAAmC,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,gBAAgB,GAAG,iBAAiB,IAAI,IAAI,CAAC;IACnD,MAAM,UAAU,GAAG,WAAW,IAAI,IAAI,CAAC;IAEvC,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE;QACvD,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QAC7C,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,GAAG,CAAC,gBAAgB,CAAC,CAAC;YACpB;gBACE,eAAe,EAAE,IAAI,CAAC,eAAe;aACtC;YACH,CAAC,CAAC,IAAI,CAAC;KACR,CAAC,CAAC;IACH,MAAM,aAAa,GAAG,gBAAgB,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3G,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC;IAE7B,MAAM,cAAc,GAA4B,EAAE,CAAC;IAEnD,IAAI,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;QACvF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,oBAAoB,GACxB,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,aAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAE/E,SAAS,YAAY,CAAC,OAAqB;gBACzC,OAAO,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE;oBACtC,MAAM;oBACN,QAAQ;oBACR,OAAO;oBACP,GAAG,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAChD,CAAC,CAAC;YACL,CAAC;YAED,IAAI,aAAa,EAAE,CAAC;gBAClB,cAAc,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;YAC/D,CAAC;iBAAM,CAAC;gBACN,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;IACH,CAAC;IACD,yCAAyC;IACzC,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAElC,MAAM,MAAM,GAAG,GAAG,SAAS,CAAC,UAAU,aAAa,IAAI,CAAC,SAAS,SAAS,GAAG,CAAC,EAAE,EAAE,CAAC;IAEnF,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AACrC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "scorecard-ai",
3
- "version": "1.0.0-alpha.8",
3
+ "version": "1.0.0",
4
4
  "description": "The official TypeScript library for the Scorecard API",
5
5
  "author": "Scorecard <team@scorecard.io>",
6
6
  "types": "./index.d.ts",
@@ -22,9 +22,6 @@
22
22
  "fix": "./scripts/format"
23
23
  },
24
24
  "dependencies": {},
25
- "resolutions": {
26
- "synckit": "0.8.8"
27
- },
28
25
  "imports": {
29
26
  "scorecard-ai": ".",
30
27
  "scorecard-ai/*": "./src/*"
@@ -1,10 +1,10 @@
1
1
  export * from "./shared.mjs";
2
+ export { Metrics, type Metric, type MetricCreateParams } from "./metrics.mjs";
2
3
  export { Projects, type Project, type ProjectCreateParams, type ProjectListParams, type ProjectsPaginatedResponse, } from "./projects.mjs";
3
4
  export { Records, type Record, type RecordCreateParams } from "./records.mjs";
4
5
  export { Runs, type Run, type RunCreateParams } from "./runs.mjs";
5
6
  export { Scores, type Score, type ScoreUpsertParams } from "./scores.mjs";
6
- export { SystemConfigs, type SystemConfig, type SystemConfigCreateParams, type SystemConfigListParams, type SystemConfigGetParams, type SystemConfigsPaginatedResponse, } from "./system-configs.mjs";
7
- export { Systems, type System, type SystemDeleteResponse, type SystemCreateParams, type SystemUpdateParams, type SystemListParams, type SystemsPaginatedResponse, } from "./systems.mjs";
7
+ export { Systems, type System, type SystemDeleteResponse, type SystemCreateParams, type SystemUpdateParams, type SystemListParams, type SystemsPaginatedResponse, } from "./systems/systems.mjs";
8
8
  export { Testcases, type Testcase, type TestcaseCreateResponse, type TestcaseDeleteResponse, type TestcaseCreateParams, type TestcaseUpdateParams, type TestcaseListParams, type TestcaseDeleteParams, type TestcasesPaginatedResponse, } from "./testcases.mjs";
9
9
  export { Testsets, type Testset, type TestsetDeleteResponse, type TestsetCreateParams, type TestsetUpdateParams, type TestsetListParams, type TestsetsPaginatedResponse, } from "./testsets.mjs";
10
10
  //# sourceMappingURL=index.d.mts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.mts","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":";OAGO,EACL,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC/B;OACM,EAAE,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,kBAAkB,EAAE;OACjD,EAAE,IAAI,EAAE,KAAK,GAAG,EAAE,KAAK,eAAe,EAAE;OACxC,EAAE,MAAM,EAAE,KAAK,KAAK,EAAE,KAAK,iBAAiB,EAAE;OAC9C,EACL,aAAa,EACb,KAAK,YAAY,EACjB,KAAK,wBAAwB,EAC7B,KAAK,sBAAsB,EAC3B,KAAK,qBAAqB,EAC1B,KAAK,8BAA8B,GACpC;OACM,EACL,OAAO,EACP,KAAK,MAAM,EACX,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,GAC9B;OACM,EACL,SAAS,EACT,KAAK,QAAQ,EACb,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAC3B,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,GAChC;OACM,EACL,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,qBAAqB,EAC1B,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC/B"}
1
+ {"version":3,"file":"index.d.mts","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":";OAGO,EAAE,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,kBAAkB,EAAE;OACjD,EACL,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC/B;OACM,EAAE,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,kBAAkB,EAAE;OACjD,EAAE,IAAI,EAAE,KAAK,GAAG,EAAE,KAAK,eAAe,EAAE;OACxC,EAAE,MAAM,EAAE,KAAK,KAAK,EAAE,KAAK,iBAAiB,EAAE;OAC9C,EACL,OAAO,EACP,KAAK,MAAM,EACX,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,GAC9B;OACM,EACL,SAAS,EACT,KAAK,QAAQ,EACb,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAC3B,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,GAChC;OACM,EACL,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,qBAAqB,EAC1B,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC/B"}
@@ -1,10 +1,10 @@
1
1
  export * from "./shared.js";
2
+ export { Metrics, type Metric, type MetricCreateParams } from "./metrics.js";
2
3
  export { Projects, type Project, type ProjectCreateParams, type ProjectListParams, type ProjectsPaginatedResponse, } from "./projects.js";
3
4
  export { Records, type Record, type RecordCreateParams } from "./records.js";
4
5
  export { Runs, type Run, type RunCreateParams } from "./runs.js";
5
6
  export { Scores, type Score, type ScoreUpsertParams } from "./scores.js";
6
- export { SystemConfigs, type SystemConfig, type SystemConfigCreateParams, type SystemConfigListParams, type SystemConfigGetParams, type SystemConfigsPaginatedResponse, } from "./system-configs.js";
7
- export { Systems, type System, type SystemDeleteResponse, type SystemCreateParams, type SystemUpdateParams, type SystemListParams, type SystemsPaginatedResponse, } from "./systems.js";
7
+ export { Systems, type System, type SystemDeleteResponse, type SystemCreateParams, type SystemUpdateParams, type SystemListParams, type SystemsPaginatedResponse, } from "./systems/systems.js";
8
8
  export { Testcases, type Testcase, type TestcaseCreateResponse, type TestcaseDeleteResponse, type TestcaseCreateParams, type TestcaseUpdateParams, type TestcaseListParams, type TestcaseDeleteParams, type TestcasesPaginatedResponse, } from "./testcases.js";
9
9
  export { Testsets, type Testset, type TestsetDeleteResponse, type TestsetCreateParams, type TestsetUpdateParams, type TestsetListParams, type TestsetsPaginatedResponse, } from "./testsets.js";
10
10
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":";OAGO,EACL,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC/B;OACM,EAAE,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,kBAAkB,EAAE;OACjD,EAAE,IAAI,EAAE,KAAK,GAAG,EAAE,KAAK,eAAe,EAAE;OACxC,EAAE,MAAM,EAAE,KAAK,KAAK,EAAE,KAAK,iBAAiB,EAAE;OAC9C,EACL,aAAa,EACb,KAAK,YAAY,EACjB,KAAK,wBAAwB,EAC7B,KAAK,sBAAsB,EAC3B,KAAK,qBAAqB,EAC1B,KAAK,8BAA8B,GACpC;OACM,EACL,OAAO,EACP,KAAK,MAAM,EACX,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,GAC9B;OACM,EACL,SAAS,EACT,KAAK,QAAQ,EACb,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAC3B,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,GAChC;OACM,EACL,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,qBAAqB,EAC1B,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC/B"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":";OAGO,EAAE,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,kBAAkB,EAAE;OACjD,EACL,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC/B;OACM,EAAE,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,kBAAkB,EAAE;OACjD,EAAE,IAAI,EAAE,KAAK,GAAG,EAAE,KAAK,eAAe,EAAE;OACxC,EAAE,MAAM,EAAE,KAAK,KAAK,EAAE,KAAK,iBAAiB,EAAE;OAC9C,EACL,OAAO,EACP,KAAK,MAAM,EACX,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,GAC9B;OACM,EACL,SAAS,EACT,KAAK,QAAQ,EACb,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAC3B,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,GAChC;OACM,EACL,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,qBAAqB,EAC1B,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC/B"}
@@ -1,9 +1,11 @@
1
1
  "use strict";
2
2
  // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
3
3
  Object.defineProperty(exports, "__esModule", { value: true });
4
- exports.Testsets = exports.Testcases = exports.Systems = exports.SystemConfigs = exports.Scores = exports.Runs = exports.Records = exports.Projects = void 0;
4
+ exports.Testsets = exports.Testcases = exports.Systems = exports.Scores = exports.Runs = exports.Records = exports.Projects = exports.Metrics = void 0;
5
5
  const tslib_1 = require("../internal/tslib.js");
6
6
  tslib_1.__exportStar(require("./shared.js"), exports);
7
+ var metrics_1 = require("./metrics.js");
8
+ Object.defineProperty(exports, "Metrics", { enumerable: true, get: function () { return metrics_1.Metrics; } });
7
9
  var projects_1 = require("./projects.js");
8
10
  Object.defineProperty(exports, "Projects", { enumerable: true, get: function () { return projects_1.Projects; } });
9
11
  var records_1 = require("./records.js");
@@ -12,9 +14,7 @@ var runs_1 = require("./runs.js");
12
14
  Object.defineProperty(exports, "Runs", { enumerable: true, get: function () { return runs_1.Runs; } });
13
15
  var scores_1 = require("./scores.js");
14
16
  Object.defineProperty(exports, "Scores", { enumerable: true, get: function () { return scores_1.Scores; } });
15
- var system_configs_1 = require("./system-configs.js");
16
- Object.defineProperty(exports, "SystemConfigs", { enumerable: true, get: function () { return system_configs_1.SystemConfigs; } });
17
- var systems_1 = require("./systems.js");
17
+ var systems_1 = require("./systems/systems.js");
18
18
  Object.defineProperty(exports, "Systems", { enumerable: true, get: function () { return systems_1.Systems; } });
19
19
  var testcases_1 = require("./testcases.js");
20
20
  Object.defineProperty(exports, "Testcases", { enumerable: true, get: function () { return testcases_1.Testcases; } });
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;;AAEtF,sDAAyB;AACzB,0CAMoB;AALlB,oGAAA,QAAQ,OAAA;AAMV,wCAA0E;AAAjE,kGAAA,OAAO,OAAA;AAChB,kCAA8D;AAArD,4FAAA,IAAI,OAAA;AACb,sCAAsE;AAA7D,gGAAA,MAAM,OAAA;AACf,sDAO0B;AANxB,+GAAA,aAAa,OAAA;AAOf,wCAQmB;AAPjB,kGAAA,OAAO,OAAA;AAQT,4CAUqB;AATnB,sGAAA,SAAS,OAAA;AAUX,0CAQoB;AAPlB,oGAAA,QAAQ,OAAA"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;;AAEtF,sDAAyB;AACzB,wCAA0E;AAAjE,kGAAA,OAAO,OAAA;AAChB,0CAMoB;AALlB,oGAAA,QAAQ,OAAA;AAMV,wCAA0E;AAAjE,kGAAA,OAAO,OAAA;AAChB,kCAA8D;AAArD,4FAAA,IAAI,OAAA;AACb,sCAAsE;AAA7D,gGAAA,MAAM,OAAA;AACf,gDAQ2B;AAPzB,kGAAA,OAAO,OAAA;AAQT,4CAUqB;AATnB,sGAAA,SAAS,OAAA;AAUX,0CAQoB;AAPlB,oGAAA,QAAQ,OAAA"}
@@ -1,11 +1,11 @@
1
1
  // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
2
2
  export * from "./shared.mjs";
3
+ export { Metrics } from "./metrics.mjs";
3
4
  export { Projects, } from "./projects.mjs";
4
5
  export { Records } from "./records.mjs";
5
6
  export { Runs } from "./runs.mjs";
6
7
  export { Scores } from "./scores.mjs";
7
- export { SystemConfigs, } from "./system-configs.mjs";
8
- export { Systems, } from "./systems.mjs";
8
+ export { Systems, } from "./systems/systems.mjs";
9
9
  export { Testcases, } from "./testcases.mjs";
10
10
  export { Testsets, } from "./testsets.mjs";
11
11
  //# sourceMappingURL=index.mjs.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.mjs","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":"AAAA,sFAAsF;;OAG/E,EACL,QAAQ,GAKT;OACM,EAAE,OAAO,EAAwC;OACjD,EAAE,IAAI,EAAkC;OACxC,EAAE,MAAM,EAAsC;OAC9C,EACL,aAAa,GAMd;OACM,EACL,OAAO,GAOR;OACM,EACL,SAAS,GASV;OACM,EACL,QAAQ,GAOT"}
1
+ {"version":3,"file":"index.mjs","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":"AAAA,sFAAsF;;OAG/E,EAAE,OAAO,EAAwC;OACjD,EACL,QAAQ,GAKT;OACM,EAAE,OAAO,EAAwC;OACjD,EAAE,IAAI,EAAkC;OACxC,EAAE,MAAM,EAAsC;OAC9C,EACL,OAAO,GAOR;OACM,EACL,SAAS,GASV;OACM,EACL,QAAQ,GAOT"}