scorecard-ai 1.0.0-alpha.6 → 1.0.0-alpha.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. package/CHANGELOG.md +57 -0
  2. package/README.md +15 -5
  3. package/client.d.mts +11 -4
  4. package/client.d.mts.map +1 -1
  5. package/client.d.ts +11 -4
  6. package/client.d.ts.map +1 -1
  7. package/client.js +20 -0
  8. package/client.js.map +1 -1
  9. package/client.mjs +20 -0
  10. package/client.mjs.map +1 -1
  11. package/internal/headers.js +1 -1
  12. package/internal/headers.js.map +1 -1
  13. package/internal/headers.mjs +1 -1
  14. package/internal/headers.mjs.map +1 -1
  15. package/internal/to-file.d.mts +0 -1
  16. package/internal/to-file.d.mts.map +1 -1
  17. package/internal/to-file.d.ts +0 -1
  18. package/internal/to-file.d.ts.map +1 -1
  19. package/internal/to-file.js +3 -2
  20. package/internal/to-file.js.map +1 -1
  21. package/internal/to-file.mjs +3 -2
  22. package/internal/to-file.mjs.map +1 -1
  23. package/internal/uploads.d.mts +1 -1
  24. package/internal/uploads.d.mts.map +1 -1
  25. package/internal/uploads.d.ts +1 -1
  26. package/internal/uploads.d.ts.map +1 -1
  27. package/internal/uploads.js +14 -4
  28. package/internal/uploads.js.map +1 -1
  29. package/internal/uploads.mjs +12 -3
  30. package/internal/uploads.mjs.map +1 -1
  31. package/internal/utils/uuid.d.mts +1 -1
  32. package/internal/utils/uuid.d.mts.map +1 -1
  33. package/internal/utils/uuid.d.ts +1 -1
  34. package/internal/utils/uuid.d.ts.map +1 -1
  35. package/internal/utils/uuid.js +1 -2
  36. package/internal/utils/uuid.js.map +1 -1
  37. package/internal/utils/uuid.mjs +1 -2
  38. package/internal/utils/uuid.mjs.map +1 -1
  39. package/lib/runAndEvaluate.d.mts.map +1 -1
  40. package/lib/runAndEvaluate.d.ts.map +1 -1
  41. package/lib/runAndEvaluate.js +1 -5
  42. package/lib/runAndEvaluate.js.map +1 -1
  43. package/lib/runAndEvaluate.mjs +1 -5
  44. package/lib/runAndEvaluate.mjs.map +1 -1
  45. package/package.json +1 -5
  46. package/resources/index.d.mts +3 -2
  47. package/resources/index.d.mts.map +1 -1
  48. package/resources/index.d.ts +3 -2
  49. package/resources/index.d.ts.map +1 -1
  50. package/resources/index.js +3 -1
  51. package/resources/index.js.map +1 -1
  52. package/resources/index.mjs +1 -0
  53. package/resources/index.mjs.map +1 -1
  54. package/resources/projects.d.mts +39 -4
  55. package/resources/projects.d.mts.map +1 -1
  56. package/resources/projects.d.ts +39 -4
  57. package/resources/projects.d.ts.map +1 -1
  58. package/resources/projects.js +23 -4
  59. package/resources/projects.js.map +1 -1
  60. package/resources/projects.mjs +23 -4
  61. package/resources/projects.mjs.map +1 -1
  62. package/resources/records.d.mts +20 -8
  63. package/resources/records.d.mts.map +1 -1
  64. package/resources/records.d.ts +20 -8
  65. package/resources/records.d.ts.map +1 -1
  66. package/resources/records.js +12 -0
  67. package/resources/records.js.map +1 -1
  68. package/resources/records.mjs +12 -0
  69. package/resources/records.mjs.map +1 -1
  70. package/resources/runs.d.mts +10 -21
  71. package/resources/runs.d.mts.map +1 -1
  72. package/resources/runs.d.ts +10 -21
  73. package/resources/runs.d.ts.map +1 -1
  74. package/resources/runs.js +9 -6
  75. package/resources/runs.js.map +1 -1
  76. package/resources/runs.mjs +9 -6
  77. package/resources/runs.mjs.map +1 -1
  78. package/resources/scores.d.mts +83 -0
  79. package/resources/scores.d.mts.map +1 -0
  80. package/resources/scores.d.ts +83 -0
  81. package/resources/scores.d.ts.map +1 -0
  82. package/resources/scores.js +35 -0
  83. package/resources/scores.js.map +1 -0
  84. package/resources/scores.mjs +31 -0
  85. package/resources/scores.mjs.map +1 -0
  86. package/resources/system-configs.d.mts +33 -0
  87. package/resources/system-configs.d.mts.map +1 -1
  88. package/resources/system-configs.d.ts +33 -0
  89. package/resources/system-configs.d.ts.map +1 -1
  90. package/resources/system-configs.js +33 -0
  91. package/resources/system-configs.js.map +1 -1
  92. package/resources/system-configs.mjs +33 -0
  93. package/resources/system-configs.mjs.map +1 -1
  94. package/resources/systems.d.mts +73 -0
  95. package/resources/systems.d.mts.map +1 -1
  96. package/resources/systems.d.ts +73 -0
  97. package/resources/systems.d.ts.map +1 -1
  98. package/resources/systems.js +73 -0
  99. package/resources/systems.js.map +1 -1
  100. package/resources/systems.mjs +73 -0
  101. package/resources/systems.mjs.map +1 -1
  102. package/resources/testcases.d.mts +72 -18
  103. package/resources/testcases.d.mts.map +1 -1
  104. package/resources/testcases.d.ts +72 -18
  105. package/resources/testcases.d.ts.map +1 -1
  106. package/resources/testcases.js +62 -0
  107. package/resources/testcases.js.map +1 -1
  108. package/resources/testcases.mjs +62 -0
  109. package/resources/testcases.mjs.map +1 -1
  110. package/resources/testsets.d.mts +78 -30
  111. package/resources/testsets.d.mts.map +1 -1
  112. package/resources/testsets.d.ts +78 -30
  113. package/resources/testsets.d.ts.map +1 -1
  114. package/resources/testsets.js +48 -0
  115. package/resources/testsets.js.map +1 -1
  116. package/resources/testsets.mjs +48 -0
  117. package/resources/testsets.mjs.map +1 -1
  118. package/src/client.ts +31 -12
  119. package/src/internal/headers.ts +1 -1
  120. package/src/internal/to-file.ts +4 -2
  121. package/src/internal/uploads.ts +16 -4
  122. package/src/internal/utils/uuid.ts +1 -3
  123. package/src/lib/runAndEvaluate.ts +1 -6
  124. package/src/resources/index.ts +5 -3
  125. package/src/resources/projects.ts +48 -9
  126. package/src/resources/records.ts +22 -10
  127. package/src/resources/runs.ts +10 -46
  128. package/src/resources/scores.ts +98 -0
  129. package/src/resources/system-configs.ts +33 -0
  130. package/src/resources/systems.ts +73 -0
  131. package/src/resources/testcases.ts +73 -19
  132. package/src/resources/testsets.ts +78 -30
  133. package/src/version.ts +1 -1
  134. package/version.d.mts +1 -1
  135. package/version.d.ts +1 -1
  136. package/version.js +1 -1
  137. package/version.mjs +1 -1
  138. package/internal/shims/crypto.d.mts +0 -13
  139. package/internal/shims/crypto.d.mts.map +0 -1
  140. package/internal/shims/crypto.d.ts +0 -13
  141. package/internal/shims/crypto.d.ts.map +0 -1
  142. package/internal/shims/crypto.js +0 -13
  143. package/internal/shims/crypto.js.map +0 -1
  144. package/internal/shims/crypto.mjs +0 -9
  145. package/internal/shims/crypto.mjs.map +0 -1
  146. package/internal/shims/file.d.mts +0 -22
  147. package/internal/shims/file.d.mts.map +0 -1
  148. package/internal/shims/file.d.ts +0 -22
  149. package/internal/shims/file.d.ts.map +0 -1
  150. package/internal/shims/file.js +0 -16
  151. package/internal/shims/file.js.map +0 -1
  152. package/internal/shims/file.mjs +0 -12
  153. package/internal/shims/file.mjs.map +0 -1
  154. package/internal/shims/getBuiltinModule.d.mts +0 -11
  155. package/internal/shims/getBuiltinModule.d.mts.map +0 -1
  156. package/internal/shims/getBuiltinModule.d.ts +0 -11
  157. package/internal/shims/getBuiltinModule.d.ts.map +0 -1
  158. package/internal/shims/getBuiltinModule.js +0 -71
  159. package/internal/shims/getBuiltinModule.js.map +0 -1
  160. package/internal/shims/getBuiltinModule.mjs +0 -67
  161. package/internal/shims/getBuiltinModule.mjs.map +0 -1
  162. package/internal/shims/nullGetBuiltinModule.d.mts +0 -2
  163. package/internal/shims/nullGetBuiltinModule.d.mts.map +0 -1
  164. package/internal/shims/nullGetBuiltinModule.d.ts +0 -2
  165. package/internal/shims/nullGetBuiltinModule.d.ts.map +0 -1
  166. package/internal/shims/nullGetBuiltinModule.js +0 -5
  167. package/internal/shims/nullGetBuiltinModule.js.map +0 -1
  168. package/internal/shims/nullGetBuiltinModule.mjs +0 -2
  169. package/internal/shims/nullGetBuiltinModule.mjs.map +0 -1
  170. package/src/internal/shims/crypto.ts +0 -18
  171. package/src/internal/shims/file.ts +0 -32
  172. package/src/internal/shims/getBuiltinModule.ts +0 -66
  173. package/src/internal/shims/nullGetBuiltinModule.ts +0 -1
@@ -23,6 +23,21 @@ export class SystemConfigs extends APIResource {
23
23
  * - Validation errors indicate fields that don't match the schema but don't
24
24
  * prevent creation
25
25
  * - Having validation errors may affect how some evaluation metrics are calculated
26
+ *
27
+ * @example
28
+ * ```ts
29
+ * const systemConfig = await client.systemConfigs.create(
30
+ * '12345678-0a8b-4f66-b6f3-2ddcfa097257',
31
+ * {
32
+ * config: {
33
+ * temperature: 0.1,
34
+ * maxTokens: 1024,
35
+ * model: 'gpt-4-turbo',
36
+ * },
37
+ * name: 'Production (Low Temperature)',
38
+ * },
39
+ * );
40
+ * ```
26
41
  */
27
42
  create(
28
43
  systemID: string,
@@ -37,6 +52,16 @@ export class SystemConfigs extends APIResource {
37
52
  *
38
53
  * System configurations provide concrete parameter values for a System Under Test,
39
54
  * defining exactly how the system should be configured during an evaluation run.
55
+ *
56
+ * @example
57
+ * ```ts
58
+ * // Automatically fetches more pages as needed.
59
+ * for await (const systemConfig of client.systemConfigs.list(
60
+ * '12345678-0a8b-4f66-b6f3-2ddcfa097257',
61
+ * )) {
62
+ * // ...
63
+ * }
64
+ * ```
40
65
  */
41
66
  list(
42
67
  systemID: string,
@@ -51,6 +76,14 @@ export class SystemConfigs extends APIResource {
51
76
 
52
77
  /**
53
78
  * Retrieve a specific system configuration by ID.
79
+ *
80
+ * @example
81
+ * ```ts
82
+ * const systemConfig = await client.systemConfigs.get(
83
+ * '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
84
+ * { systemId: '12345678-0a8b-4f66-b6f3-2ddcfa097257' },
85
+ * );
86
+ * ```
54
87
  */
55
88
  get(
56
89
  systemConfigID: string,
@@ -23,6 +23,45 @@ export class Systems extends APIResource {
23
23
  *
24
24
  * This separation lets you evaluate any system as a black box, focusing on its
25
25
  * interface rather than implementation details.
26
+ *
27
+ * @example
28
+ * ```ts
29
+ * const system = await client.systems.create('314', {
30
+ * configSchema: {
31
+ * type: 'object',
32
+ * properties: {
33
+ * temperature: { type: 'number' },
34
+ * maxTokens: { type: 'integer' },
35
+ * model: { type: 'string', enum: ['gpt-4', 'gpt-4-turbo'] },
36
+ * },
37
+ * required: ['model'],
38
+ * },
39
+ * description: 'Production chatbot powered by GPT-4',
40
+ * inputSchema: {
41
+ * type: 'object',
42
+ * properties: {
43
+ * messages: {
44
+ * type: 'array',
45
+ * items: {
46
+ * type: 'object',
47
+ * properties: {
48
+ * role: { type: 'string', enum: ['system', 'user', 'assistant'] },
49
+ * content: { type: 'string' },
50
+ * },
51
+ * required: ['role', 'content'],
52
+ * },
53
+ * },
54
+ * },
55
+ * required: ['messages'],
56
+ * },
57
+ * name: 'GPT-4 Chatbot',
58
+ * outputSchema: {
59
+ * type: 'object',
60
+ * properties: { response: { type: 'string' } },
61
+ * required: ['response'],
62
+ * },
63
+ * });
64
+ * ```
26
65
  */
27
66
  create(projectID: string, body: SystemCreateParams, options?: RequestOptions): APIPromise<System> {
28
67
  return this._client.post(path`/projects/${projectID}/systems`, { body, ...options });
@@ -41,6 +80,18 @@ export class Systems extends APIResource {
41
80
  * - Schema updates won't invalidate existing evaluations or configurations
42
81
  * - For significant redesigns, creating a new system definition provides a cleaner
43
82
  * separation
83
+ *
84
+ * @example
85
+ * ```ts
86
+ * const system = await client.systems.update(
87
+ * '12345678-0a8b-4f66-b6f3-2ddcfa097257',
88
+ * {
89
+ * description:
90
+ * 'Updated production chatbot powered by GPT-4 Turbo',
91
+ * name: 'GPT-4 Turbo Chatbot',
92
+ * },
93
+ * );
94
+ * ```
44
95
  */
45
96
  update(
46
97
  systemID: string,
@@ -52,6 +103,14 @@ export class Systems extends APIResource {
52
103
 
53
104
  /**
54
105
  * Retrieve a paginated list of all systems. Systems are ordered by creation date.
106
+ *
107
+ * @example
108
+ * ```ts
109
+ * // Automatically fetches more pages as needed.
110
+ * for await (const system of client.systems.list('314')) {
111
+ * // ...
112
+ * }
113
+ * ```
55
114
  */
56
115
  list(
57
116
  projectID: string,
@@ -67,6 +126,13 @@ export class Systems extends APIResource {
67
126
  /**
68
127
  * Delete a system definition by ID. This will not delete associated system
69
128
  * configurations.
129
+ *
130
+ * @example
131
+ * ```ts
132
+ * const system = await client.systems.delete(
133
+ * '12345678-0a8b-4f66-b6f3-2ddcfa097257',
134
+ * );
135
+ * ```
70
136
  */
71
137
  delete(systemID: string, options?: RequestOptions): APIPromise<SystemDeleteResponse> {
72
138
  return this._client.delete(path`/systems/${systemID}`, options);
@@ -74,6 +140,13 @@ export class Systems extends APIResource {
74
140
 
75
141
  /**
76
142
  * Retrieve a specific system by ID.
143
+ *
144
+ * @example
145
+ * ```ts
146
+ * const system = await client.systems.get(
147
+ * '12345678-0a8b-4f66-b6f3-2ddcfa097257',
148
+ * );
149
+ * ```
77
150
  */
78
151
  get(systemID: string, options?: RequestOptions): APIPromise<System> {
79
152
  return this._client.get(path`/systems/${systemID}`, options);
@@ -9,6 +9,37 @@ import { path } from '../internal/utils/path';
9
9
  export class Testcases extends APIResource {
10
10
  /**
11
11
  * Create multiple Testcases in the specified Testset.
12
+ *
13
+ * @example
14
+ * ```ts
15
+ * const testcase = await client.testcases.create('246', {
16
+ * items: [
17
+ * {
18
+ * jsonData: {
19
+ * question: 'What is the capital of France?',
20
+ * idealAnswer: 'Paris',
21
+ * provenance: 'hand_curated',
22
+ * },
23
+ * },
24
+ * {
25
+ * jsonData: {
26
+ * question:
27
+ * 'What is the largest planet in our solar system?',
28
+ * idealAnswer: 'Jupiter',
29
+ * provenance: 'synthetic',
30
+ * },
31
+ * },
32
+ * {
33
+ * jsonData: {
34
+ * question:
35
+ * 'How many planets are in our solar system?',
36
+ * idealAnswer: 8,
37
+ * provenance: 'user_feedback',
38
+ * },
39
+ * },
40
+ * ],
41
+ * });
42
+ * ```
12
43
  */
13
44
  create(
14
45
  testsetID: string,
@@ -20,6 +51,17 @@ export class Testcases extends APIResource {
20
51
 
21
52
  /**
22
53
  * Replace the data of an existing Testcase while keeping its ID.
54
+ *
55
+ * @example
56
+ * ```ts
57
+ * const testcase = await client.testcases.update('248', {
58
+ * jsonData: {
59
+ * question: 'What is the capital of France?',
60
+ * idealAnswer: 'Paris is the capital of France',
61
+ * provenance: 'hand_curated',
62
+ * },
63
+ * });
64
+ * ```
23
65
  */
24
66
  update(testcaseID: string, body: TestcaseUpdateParams, options?: RequestOptions): APIPromise<Testcase> {
25
67
  return this._client.put(path`/testcases/${testcaseID}`, { body, ...options });
@@ -27,6 +69,14 @@ export class Testcases extends APIResource {
27
69
 
28
70
  /**
29
71
  * Retrieve a paginated list of Testcases belonging to a Testset.
72
+ *
73
+ * @example
74
+ * ```ts
75
+ * // Automatically fetches more pages as needed.
76
+ * for await (const testcase of client.testcases.list('246')) {
77
+ * // ...
78
+ * }
79
+ * ```
30
80
  */
31
81
  list(
32
82
  testsetID: string,
@@ -41,6 +91,13 @@ export class Testcases extends APIResource {
41
91
 
42
92
  /**
43
93
  * Delete multiple Testcases by their IDs.
94
+ *
95
+ * @example
96
+ * ```ts
97
+ * const testcase = await client.testcases.delete({
98
+ * ids: ['123', '124', '125'],
99
+ * });
100
+ * ```
44
101
  */
45
102
  delete(body: TestcaseDeleteParams, options?: RequestOptions): APIPromise<TestcaseDeleteResponse> {
46
103
  return this._client.post('/testcases/bulk-delete', { body, ...options });
@@ -48,6 +105,11 @@ export class Testcases extends APIResource {
48
105
 
49
106
  /**
50
107
  * Retrieve a specific Testcase by ID.
108
+ *
109
+ * @example
110
+ * ```ts
111
+ * const testcase = await client.testcases.get('248');
112
+ * ```
51
113
  */
52
114
  get(testcaseID: string, options?: RequestOptions): APIPromise<Testcase> {
53
115
  return this._client.get(path`/testcases/${testcaseID}`, options);
@@ -58,11 +120,11 @@ export type TestcasesPaginatedResponse = PaginatedResponse<Testcase>;
58
120
 
59
121
  /**
60
122
  * A test case in the Scorecard system. Contains JSON data that is validated
61
- * against the schema defined by its Testset. The `inputs` and `labels` fields are
62
- * derived from the `data` field based on the Testset's `fieldMapping`, and include
63
- * all mapped fields, including those with validation errors. Testcases are stored
64
- * regardless of validation results, with any validation errors included in the
65
- * `validationErrors` field.
123
+ * against the schema defined by its Testset. The `inputs` and `expected` fields
124
+ * are derived from the `data` field based on the Testset's `fieldMapping`, and
125
+ * include all mapped fields, including those with validation errors. Testcases are
126
+ * stored regardless of validation results, with any validation errors included in
127
+ * the `validationErrors` field.
66
128
  */
67
129
  export interface Testcase {
68
130
  /**
@@ -70,6 +132,12 @@ export interface Testcase {
70
132
  */
71
133
  id: string;
72
134
 
135
+ /**
136
+ * Derived from data based on the Testset's fieldMapping. Contains all fields
137
+ * marked as expected outputs, including those with validation errors.
138
+ */
139
+ expected: Record<string, unknown>;
140
+
73
141
  /**
74
142
  * Derived from data based on the Testset's fieldMapping. Contains all fields
75
143
  * marked as inputs, including those with validation errors.
@@ -81,12 +149,6 @@ export interface Testcase {
81
149
  */
82
150
  jsonData: Record<string, unknown>;
83
151
 
84
- /**
85
- * Derived from data based on the Testset's fieldMapping. Contains all fields
86
- * marked as labels, including those with validation errors.
87
- */
88
- labels: Record<string, unknown>;
89
-
90
152
  /**
91
153
  * The ID of the Testset this Testcase belongs to.
92
154
  */
@@ -132,14 +194,6 @@ export interface TestcaseCreateParams {
132
194
  }
133
195
 
134
196
  export namespace TestcaseCreateParams {
135
- /**
136
- * A test case in the Scorecard system. Contains JSON data that is validated
137
- * against the schema defined by its Testset. The `inputs` and `labels` fields are
138
- * derived from the `data` field based on the Testset's `fieldMapping`, and include
139
- * all mapped fields, including those with validation errors. Testcases are stored
140
- * regardless of validation results, with any validation errors included in the
141
- * `validationErrors` field.
142
- */
143
197
  export interface Item {
144
198
  /**
145
199
  * The JSON data of the Testcase, which is validated against the Testset's schema.
@@ -10,6 +10,28 @@ export class Testsets extends APIResource {
10
10
  /**
11
11
  * Create a new Testset for a Project. The Testset will be created in the Project
12
12
  * specified in the path.
13
+ *
14
+ * @example
15
+ * ```ts
16
+ * const testset = await client.testsets.create('314', {
17
+ * description: 'Testset for long context Q&A chatbot.',
18
+ * fieldMapping: {
19
+ * inputs: ['question'],
20
+ * expected: ['idealAnswer'],
21
+ * metadata: [],
22
+ * },
23
+ * jsonSchema: {
24
+ * type: 'object',
25
+ * properties: {
26
+ * question: { type: 'string' },
27
+ * idealAnswer: { type: 'string' },
28
+ * provenance: { type: 'string' },
29
+ * geo: { type: 'string' },
30
+ * },
31
+ * },
32
+ * name: 'Long Context Q&A',
33
+ * });
34
+ * ```
13
35
  */
14
36
  create(projectID: string, body: TestsetCreateParams, options?: RequestOptions): APIPromise<Testset> {
15
37
  return this._client.post(path`/projects/${projectID}/testsets`, { body, ...options });
@@ -28,6 +50,14 @@ export class Testsets extends APIResource {
28
50
  * updated schema
29
51
  * - For complete control, provide both schema and fieldMapping when updating the
30
52
  * schema
53
+ *
54
+ * @example
55
+ * ```ts
56
+ * const testset = await client.testsets.update('246', {
57
+ * description: 'Updated description for the Q&A Testset.',
58
+ * name: 'Updated Q&A Testset',
59
+ * });
60
+ * ```
31
61
  */
32
62
  update(
33
63
  testsetID: string,
@@ -39,6 +69,14 @@ export class Testsets extends APIResource {
39
69
 
40
70
  /**
41
71
  * Retrieve a paginated list of Testsets belonging to a Project.
72
+ *
73
+ * @example
74
+ * ```ts
75
+ * // Automatically fetches more pages as needed.
76
+ * for await (const testset of client.testsets.list('314')) {
77
+ * // ...
78
+ * }
79
+ * ```
42
80
  */
43
81
  list(
44
82
  projectID: string,
@@ -53,6 +91,11 @@ export class Testsets extends APIResource {
53
91
 
54
92
  /**
55
93
  * Delete Testset
94
+ *
95
+ * @example
96
+ * ```ts
97
+ * const testset = await client.testsets.delete('246');
98
+ * ```
56
99
  */
57
100
  delete(testsetID: string, options?: RequestOptions): APIPromise<TestsetDeleteResponse> {
58
101
  return this._client.delete(path`/testsets/${testsetID}`, options);
@@ -60,6 +103,11 @@ export class Testsets extends APIResource {
60
103
 
61
104
  /**
62
105
  * Get Testset by ID
106
+ *
107
+ * @example
108
+ * ```ts
109
+ * const testset = await client.testsets.get('246');
110
+ * ```
63
111
  */
64
112
  get(testsetID: string, options?: RequestOptions): APIPromise<Testset> {
65
113
  return this._client.get(path`/testsets/${testsetID}`, options);
@@ -71,9 +119,9 @@ export type TestsetsPaginatedResponse = PaginatedResponse<Testset>;
71
119
  /**
72
120
  * A collection of Testcases that share the same schema. Each Testset defines the
73
121
  * structure of its Testcases through a JSON schema. The `fieldMapping` object maps
74
- * top-level keys of the Testcase schema to their roles (input/label). Fields not
75
- * mentioned in the `fieldMapping` during creation or update are treated as
76
- * metadata.
122
+ * top-level keys of the Testcase schema to their roles (input/expected output).
123
+ * Fields not mentioned in the `fieldMapping` during creation or update are treated
124
+ * as metadata.
77
125
  *
78
126
  * ## JSON Schema validation constraints supported:
79
127
  *
@@ -106,8 +154,8 @@ export interface Testset {
106
154
  description: string;
107
155
 
108
156
  /**
109
- * Maps top-level keys of the Testcase schema to their roles (input/label).
110
- * Unmapped fields are treated as metadata.
157
+ * Maps top-level keys of the Testcase schema to their roles (input/expected
158
+ * output). Unmapped fields are treated as metadata.
111
159
  */
112
160
  fieldMapping: Testset.FieldMapping;
113
161
 
@@ -124,22 +172,22 @@ export interface Testset {
124
172
 
125
173
  export namespace Testset {
126
174
  /**
127
- * Maps top-level keys of the Testcase schema to their roles (input/label).
128
- * Unmapped fields are treated as metadata.
175
+ * Maps top-level keys of the Testcase schema to their roles (input/expected
176
+ * output). Unmapped fields are treated as metadata.
129
177
  */
130
178
  export interface FieldMapping {
131
179
  /**
132
- * Fields that represent inputs to the AI system.
180
+ * Fields that represent expected outputs.
133
181
  */
134
- inputs: Array<string>;
182
+ expected: Array<string>;
135
183
 
136
184
  /**
137
- * Fields that represent expected outputs/labels.
185
+ * Fields that represent inputs to the AI system.
138
186
  */
139
- labels: Array<string>;
187
+ inputs: Array<string>;
140
188
 
141
189
  /**
142
- * Fields that are not inputs or labels.
190
+ * Fields that are not inputs or expected outputs.
143
191
  */
144
192
  metadata: Array<string>;
145
193
  }
@@ -159,8 +207,8 @@ export interface TestsetCreateParams {
159
207
  description: string;
160
208
 
161
209
  /**
162
- * Maps top-level keys of the Testcase schema to their roles (input/label).
163
- * Unmapped fields are treated as metadata.
210
+ * Maps top-level keys of the Testcase schema to their roles (input/expected
211
+ * output). Unmapped fields are treated as metadata.
164
212
  */
165
213
  fieldMapping: TestsetCreateParams.FieldMapping;
166
214
 
@@ -177,22 +225,22 @@ export interface TestsetCreateParams {
177
225
 
178
226
  export namespace TestsetCreateParams {
179
227
  /**
180
- * Maps top-level keys of the Testcase schema to their roles (input/label).
181
- * Unmapped fields are treated as metadata.
228
+ * Maps top-level keys of the Testcase schema to their roles (input/expected
229
+ * output). Unmapped fields are treated as metadata.
182
230
  */
183
231
  export interface FieldMapping {
184
232
  /**
185
- * Fields that represent inputs to the AI system.
233
+ * Fields that represent expected outputs.
186
234
  */
187
- inputs: Array<string>;
235
+ expected: Array<string>;
188
236
 
189
237
  /**
190
- * Fields that represent expected outputs/labels.
238
+ * Fields that represent inputs to the AI system.
191
239
  */
192
- labels: Array<string>;
240
+ inputs: Array<string>;
193
241
 
194
242
  /**
195
- * Fields that are not inputs or labels.
243
+ * Fields that are not inputs or expected outputs.
196
244
  */
197
245
  metadata: Array<string>;
198
246
  }
@@ -205,8 +253,8 @@ export interface TestsetUpdateParams {
205
253
  description?: string;
206
254
 
207
255
  /**
208
- * Maps top-level keys of the Testcase schema to their roles (input/label).
209
- * Unmapped fields are treated as metadata.
256
+ * Maps top-level keys of the Testcase schema to their roles (input/expected
257
+ * output). Unmapped fields are treated as metadata.
210
258
  */
211
259
  fieldMapping?: TestsetUpdateParams.FieldMapping;
212
260
 
@@ -223,22 +271,22 @@ export interface TestsetUpdateParams {
223
271
 
224
272
  export namespace TestsetUpdateParams {
225
273
  /**
226
- * Maps top-level keys of the Testcase schema to their roles (input/label).
227
- * Unmapped fields are treated as metadata.
274
+ * Maps top-level keys of the Testcase schema to their roles (input/expected
275
+ * output). Unmapped fields are treated as metadata.
228
276
  */
229
277
  export interface FieldMapping {
230
278
  /**
231
- * Fields that represent inputs to the AI system.
279
+ * Fields that represent expected outputs.
232
280
  */
233
- inputs: Array<string>;
281
+ expected: Array<string>;
234
282
 
235
283
  /**
236
- * Fields that represent expected outputs/labels.
284
+ * Fields that represent inputs to the AI system.
237
285
  */
238
- labels: Array<string>;
286
+ inputs: Array<string>;
239
287
 
240
288
  /**
241
- * Fields that are not inputs or labels.
289
+ * Fields that are not inputs or expected outputs.
242
290
  */
243
291
  metadata: Array<string>;
244
292
  }
package/src/version.ts CHANGED
@@ -1 +1 @@
1
- export const VERSION = '1.0.0-alpha.6'; // x-release-please-version
1
+ export const VERSION = '1.0.0-alpha.8'; // x-release-please-version
package/version.d.mts CHANGED
@@ -1,2 +1,2 @@
1
- export declare const VERSION = "1.0.0-alpha.6";
1
+ export declare const VERSION = "1.0.0-alpha.8";
2
2
  //# sourceMappingURL=version.d.mts.map
package/version.d.ts CHANGED
@@ -1,2 +1,2 @@
1
- export declare const VERSION = "1.0.0-alpha.6";
1
+ export declare const VERSION = "1.0.0-alpha.8";
2
2
  //# sourceMappingURL=version.d.ts.map
package/version.js CHANGED
@@ -1,5 +1,5 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.VERSION = void 0;
4
- exports.VERSION = '1.0.0-alpha.6'; // x-release-please-version
4
+ exports.VERSION = '1.0.0-alpha.8'; // x-release-please-version
5
5
  //# sourceMappingURL=version.js.map
package/version.mjs CHANGED
@@ -1,2 +1,2 @@
1
- export const VERSION = '1.0.0-alpha.6'; // x-release-please-version
1
+ export const VERSION = '1.0.0-alpha.8'; // x-release-please-version
2
2
  //# sourceMappingURL=version.mjs.map
@@ -1,13 +0,0 @@
1
- type Crypto = {
2
- /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/getRandomValues) */
3
- getRandomValues<T extends ArrayBufferView | null>(array: T): T;
4
- /**
5
- * Available only in secure contexts.
6
- *
7
- * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/randomUUID)
8
- */
9
- randomUUID?: () => string;
10
- };
11
- export declare let getCrypto: () => Crypto | undefined;
12
- export {};
13
- //# sourceMappingURL=crypto.d.mts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"crypto.d.mts","sourceRoot":"","sources":["../../src/internal/shims/crypto.ts"],"names":[],"mappings":"AAEA,KAAK,MAAM,GAAG;IACZ,yFAAyF;IACzF,eAAe,CAAC,CAAC,SAAS,eAAe,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC;IAC/D;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,MAAM,CAAC;CAC3B,CAAC;AACF,eAAO,IAAI,SAAS,EAAE,MAAM,MAAM,GAAG,SAKpC,CAAC"}
@@ -1,13 +0,0 @@
1
- type Crypto = {
2
- /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/getRandomValues) */
3
- getRandomValues<T extends ArrayBufferView | null>(array: T): T;
4
- /**
5
- * Available only in secure contexts.
6
- *
7
- * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/randomUUID)
8
- */
9
- randomUUID?: () => string;
10
- };
11
- export declare let getCrypto: () => Crypto | undefined;
12
- export {};
13
- //# sourceMappingURL=crypto.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"crypto.d.ts","sourceRoot":"","sources":["../../src/internal/shims/crypto.ts"],"names":[],"mappings":"AAEA,KAAK,MAAM,GAAG;IACZ,yFAAyF;IACzF,eAAe,CAAC,CAAC,SAAS,eAAe,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC;IAC/D;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,MAAM,CAAC;CAC3B,CAAC;AACF,eAAO,IAAI,SAAS,EAAE,MAAM,MAAM,GAAG,SAKpC,CAAC"}
@@ -1,13 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.getCrypto = void 0;
4
- const getBuiltinModule_1 = require("./getBuiltinModule.js");
5
- let getCrypto = function lazyGetCrypto() {
6
- if (exports.getCrypto !== lazyGetCrypto)
7
- return (0, exports.getCrypto)();
8
- const crypto = globalThis.crypto || (0, getBuiltinModule_1.getBuiltinModule)?.('node:crypto')?.webcrypto;
9
- exports.getCrypto = () => crypto;
10
- return crypto;
11
- };
12
- exports.getCrypto = getCrypto;
13
- //# sourceMappingURL=crypto.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"crypto.js","sourceRoot":"","sources":["../../src/internal/shims/crypto.ts"],"names":[],"mappings":";;;AAAA,4DAAsD;AAY/C,IAAI,SAAS,GAA6B,SAAS,aAAa;IACrE,IAAI,iBAAS,KAAK,aAAa;QAAE,OAAO,IAAA,iBAAS,GAAE,CAAC;IACpD,MAAM,MAAM,GAAY,UAAkB,CAAC,MAAM,IAAK,IAAA,mCAAgB,CAAA,EAAE,CAAC,aAAa,CAAS,EAAE,SAAS,CAAC;IAC3G,iBAAS,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AALS,QAAA,SAAS,aAKlB"}
@@ -1,9 +0,0 @@
1
- import { getBuiltinModule } from "./getBuiltinModule.mjs";
2
- export let getCrypto = function lazyGetCrypto() {
3
- if (getCrypto !== lazyGetCrypto)
4
- return getCrypto();
5
- const crypto = globalThis.crypto || getBuiltinModule?.('node:crypto')?.webcrypto;
6
- getCrypto = () => crypto;
7
- return crypto;
8
- };
9
- //# sourceMappingURL=crypto.mjs.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"crypto.mjs","sourceRoot":"","sources":["../../src/internal/shims/crypto.ts"],"names":[],"mappings":"OAAO,EAAE,gBAAgB,EAAE;AAY3B,MAAM,CAAC,IAAI,SAAS,GAA6B,SAAS,aAAa;IACrE,IAAI,SAAS,KAAK,aAAa;QAAE,OAAO,SAAS,EAAE,CAAC;IACpD,MAAM,MAAM,GAAY,UAAkB,CAAC,MAAM,IAAK,gBAAgB,EAAE,CAAC,aAAa,CAAS,EAAE,SAAS,CAAC;IAC3G,SAAS,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC"}
@@ -1,22 +0,0 @@
1
- export declare let getFile: () => FileConstructor;
2
- type FileConstructor = typeof globalThis extends {
3
- File: infer fileConstructor;
4
- } ? fileConstructor : typeof FallbackFile;
5
- export type File = InstanceType<FileConstructor>;
6
- type FallbackBlobSource = ConstructorParameters<typeof Blob>[0] extends infer T ? T : never;
7
- /**
8
- * A [`File`](https://developer.mozilla.org/en-US/docs/Web/API/File) provides information about files.
9
- */
10
- declare class FallbackFile extends Blob {
11
- constructor(sources: FallbackBlobSource, fileName: string, options?: any);
12
- /**
13
- * The name of the `File`.
14
- */
15
- readonly name: string;
16
- /**
17
- * The last modified date of the `File`.
18
- */
19
- readonly lastModified: number;
20
- }
21
- export {};
22
- //# sourceMappingURL=file.d.mts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"file.d.mts","sourceRoot":"","sources":["../../src/internal/shims/file.ts"],"names":[],"mappings":"AAEA,eAAO,IAAI,OAAO,QAA2B,eAO5C,CAAC;AAEF,KAAK,eAAe,GAClB,OAAO,UAAU,SAAS;IAAE,IAAI,EAAE,MAAM,eAAe,CAAA;CAAE,GAAG,eAAe,GAAG,OAAO,YAAY,CAAC;AACpG,MAAM,MAAM,IAAI,GAAG,YAAY,CAAC,eAAe,CAAC,CAAC;AAIjD,KAAK,kBAAkB,GAAG,qBAAqB,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;AAC5F;;GAEG;AACH,OAAO,OAAO,YAAa,SAAQ,IAAI;gBACzB,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG;IACxE;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B"}
@@ -1,22 +0,0 @@
1
- export declare let getFile: () => FileConstructor;
2
- type FileConstructor = typeof globalThis extends {
3
- File: infer fileConstructor;
4
- } ? fileConstructor : typeof FallbackFile;
5
- export type File = InstanceType<FileConstructor>;
6
- type FallbackBlobSource = ConstructorParameters<typeof Blob>[0] extends infer T ? T : never;
7
- /**
8
- * A [`File`](https://developer.mozilla.org/en-US/docs/Web/API/File) provides information about files.
9
- */
10
- declare class FallbackFile extends Blob {
11
- constructor(sources: FallbackBlobSource, fileName: string, options?: any);
12
- /**
13
- * The name of the `File`.
14
- */
15
- readonly name: string;
16
- /**
17
- * The last modified date of the `File`.
18
- */
19
- readonly lastModified: number;
20
- }
21
- export {};
22
- //# sourceMappingURL=file.d.ts.map