scorecard-ai 1.0.0-alpha.8 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/CHANGELOG.md +46 -0
  2. package/README.md +25 -37
  3. package/client.d.mts +5 -4
  4. package/client.d.mts.map +1 -1
  5. package/client.d.ts +5 -4
  6. package/client.d.ts.map +1 -1
  7. package/client.js +19 -4
  8. package/client.js.map +1 -1
  9. package/client.mjs +19 -4
  10. package/client.mjs.map +1 -1
  11. package/core/pagination.d.mts +1 -1
  12. package/core/pagination.d.mts.map +1 -1
  13. package/core/pagination.d.ts +1 -1
  14. package/core/pagination.d.ts.map +1 -1
  15. package/index.d.mts +1 -0
  16. package/index.d.mts.map +1 -1
  17. package/index.d.ts +1 -0
  18. package/index.d.ts.map +1 -1
  19. package/index.js +3 -1
  20. package/index.js.map +1 -1
  21. package/index.mjs +1 -0
  22. package/index.mjs.map +1 -1
  23. package/internal/detect-platform.js +3 -3
  24. package/internal/detect-platform.js.map +1 -1
  25. package/internal/detect-platform.mjs +3 -3
  26. package/internal/detect-platform.mjs.map +1 -1
  27. package/internal/shim-types.d.mts +11 -22
  28. package/internal/shim-types.d.mts.map +1 -0
  29. package/internal/shim-types.d.ts +11 -22
  30. package/internal/shim-types.d.ts.map +1 -0
  31. package/internal/shim-types.js +4 -0
  32. package/internal/shim-types.js.map +1 -0
  33. package/internal/shim-types.mjs +3 -0
  34. package/internal/shim-types.mjs.map +1 -0
  35. package/internal/shims.d.mts +2 -2
  36. package/internal/shims.d.mts.map +1 -1
  37. package/internal/shims.d.ts +2 -2
  38. package/internal/shims.d.ts.map +1 -1
  39. package/internal/tslib.js +6 -6
  40. package/internal/uploads.js.map +1 -1
  41. package/internal/uploads.mjs.map +1 -1
  42. package/lib/runAndEvaluate.d.mts +62 -10
  43. package/lib/runAndEvaluate.d.mts.map +1 -1
  44. package/lib/runAndEvaluate.d.ts +62 -10
  45. package/lib/runAndEvaluate.d.ts.map +1 -1
  46. package/lib/runAndEvaluate.js +72 -19
  47. package/lib/runAndEvaluate.js.map +1 -1
  48. package/lib/runAndEvaluate.mjs +72 -19
  49. package/lib/runAndEvaluate.mjs.map +1 -1
  50. package/package.json +1 -4
  51. package/resources/index.d.mts +2 -2
  52. package/resources/index.d.mts.map +1 -1
  53. package/resources/index.d.ts +2 -2
  54. package/resources/index.d.ts.map +1 -1
  55. package/resources/index.js +4 -4
  56. package/resources/index.js.map +1 -1
  57. package/resources/index.mjs +2 -2
  58. package/resources/index.mjs.map +1 -1
  59. package/resources/metrics.d.mts +421 -0
  60. package/resources/metrics.d.mts.map +1 -0
  61. package/resources/metrics.d.ts +421 -0
  62. package/resources/metrics.d.ts.map +1 -0
  63. package/resources/metrics.js +33 -0
  64. package/resources/metrics.js.map +1 -0
  65. package/resources/metrics.mjs +29 -0
  66. package/resources/metrics.mjs.map +1 -0
  67. package/resources/runs.d.mts +8 -8
  68. package/resources/runs.d.mts.map +1 -1
  69. package/resources/runs.d.ts +8 -8
  70. package/resources/runs.d.ts.map +1 -1
  71. package/resources/runs.js +1 -1
  72. package/resources/runs.mjs +1 -1
  73. package/resources/systems/index.d.mts +3 -0
  74. package/resources/systems/index.d.mts.map +1 -0
  75. package/resources/systems/index.d.ts +3 -0
  76. package/resources/systems/index.d.ts.map +1 -0
  77. package/resources/systems/index.js +9 -0
  78. package/resources/systems/index.js.map +1 -0
  79. package/resources/systems/index.mjs +4 -0
  80. package/resources/systems/index.mjs.map +1 -0
  81. package/resources/systems/systems.d.mts +229 -0
  82. package/resources/systems/systems.d.mts.map +1 -0
  83. package/resources/systems/systems.d.ts +229 -0
  84. package/resources/systems/systems.d.ts.map +1 -0
  85. package/resources/systems/systems.js +151 -0
  86. package/resources/systems/systems.js.map +1 -0
  87. package/resources/systems/systems.mjs +146 -0
  88. package/resources/systems/systems.mjs.map +1 -0
  89. package/resources/systems/versions.d.mts +132 -0
  90. package/resources/systems/versions.d.mts.map +1 -0
  91. package/resources/systems/versions.d.ts +132 -0
  92. package/resources/systems/versions.d.ts.map +1 -0
  93. package/resources/systems/versions.js +82 -0
  94. package/resources/systems/versions.js.map +1 -0
  95. package/resources/systems/versions.mjs +78 -0
  96. package/resources/systems/versions.mjs.map +1 -0
  97. package/resources/systems.d.mts +1 -224
  98. package/resources/systems.d.mts.map +1 -1
  99. package/resources/systems.d.ts +1 -224
  100. package/resources/systems.d.ts.map +1 -1
  101. package/resources/systems.js +2 -139
  102. package/resources/systems.js.map +1 -1
  103. package/resources/systems.mjs +1 -137
  104. package/resources/systems.mjs.map +1 -1
  105. package/resources/testsets.d.mts +1 -1
  106. package/resources/testsets.d.ts +1 -1
  107. package/resources/testsets.js +1 -1
  108. package/resources/testsets.mjs +1 -1
  109. package/src/client.ts +28 -28
  110. package/src/core/pagination.ts +1 -1
  111. package/src/index.ts +2 -0
  112. package/src/internal/detect-platform.ts +3 -3
  113. package/src/internal/shim-types.ts +26 -0
  114. package/src/internal/shims.ts +2 -2
  115. package/src/internal/uploads.ts +1 -1
  116. package/src/lib/runAndEvaluate.ts +159 -31
  117. package/src/resources/index.ts +2 -9
  118. package/src/resources/metrics.ts +525 -0
  119. package/src/resources/runs.ts +8 -8
  120. package/src/resources/systems/index.ts +18 -0
  121. package/src/resources/systems/systems.ts +299 -0
  122. package/src/resources/systems/versions.ts +166 -0
  123. package/src/resources/systems.ts +1 -277
  124. package/src/resources/testsets.ts +1 -1
  125. package/src/version.ts +1 -1
  126. package/version.d.mts +1 -1
  127. package/version.d.mts.map +1 -1
  128. package/version.d.ts +1 -1
  129. package/version.d.ts.map +1 -1
  130. package/version.js +1 -1
  131. package/version.js.map +1 -1
  132. package/version.mjs +1 -1
  133. package/version.mjs.map +1 -1
  134. package/resources/system-configs.d.mts +0 -155
  135. package/resources/system-configs.d.mts.map +0 -1
  136. package/resources/system-configs.d.ts +0 -155
  137. package/resources/system-configs.d.ts.map +0 -1
  138. package/resources/system-configs.js +0 -83
  139. package/resources/system-configs.js.map +0 -1
  140. package/resources/system-configs.mjs +0 -79
  141. package/resources/system-configs.mjs.map +0 -1
  142. package/src/internal/shim-types.d.ts +0 -28
  143. package/src/resources/system-configs.ts +0 -203
@@ -1,279 +1,3 @@
1
1
  // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
2
2
 
3
- import { APIResource } from '../core/resource';
4
- import { APIPromise } from '../core/api-promise';
5
- import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../core/pagination';
6
- import { RequestOptions } from '../internal/request-options';
7
- import { path } from '../internal/utils/path';
8
-
9
- export class Systems extends APIResource {
10
- /**
11
- * Create a new system definition that specifies the interface contracts for a
12
- * component you want to evaluate.
13
- *
14
- * A system acts as a template that defines three key contracts through JSON
15
- * Schemas:
16
- *
17
- * 1. Input Schema: What data your system accepts (e.g., user queries, context
18
- * documents)
19
- * 2. Output Schema: What data your system produces (e.g., responses, confidence
20
- * scores)
21
- * 3. Config Schema: What parameters can be adjusted (e.g., model selection,
22
- * temperature)
23
- *
24
- * This separation lets you evaluate any system as a black box, focusing on its
25
- * interface rather than implementation details.
26
- *
27
- * @example
28
- * ```ts
29
- * const system = await client.systems.create('314', {
30
- * configSchema: {
31
- * type: 'object',
32
- * properties: {
33
- * temperature: { type: 'number' },
34
- * maxTokens: { type: 'integer' },
35
- * model: { type: 'string', enum: ['gpt-4', 'gpt-4-turbo'] },
36
- * },
37
- * required: ['model'],
38
- * },
39
- * description: 'Production chatbot powered by GPT-4',
40
- * inputSchema: {
41
- * type: 'object',
42
- * properties: {
43
- * messages: {
44
- * type: 'array',
45
- * items: {
46
- * type: 'object',
47
- * properties: {
48
- * role: { type: 'string', enum: ['system', 'user', 'assistant'] },
49
- * content: { type: 'string' },
50
- * },
51
- * required: ['role', 'content'],
52
- * },
53
- * },
54
- * },
55
- * required: ['messages'],
56
- * },
57
- * name: 'GPT-4 Chatbot',
58
- * outputSchema: {
59
- * type: 'object',
60
- * properties: { response: { type: 'string' } },
61
- * required: ['response'],
62
- * },
63
- * });
64
- * ```
65
- */
66
- create(projectID: string, body: SystemCreateParams, options?: RequestOptions): APIPromise<System> {
67
- return this._client.post(path`/projects/${projectID}/systems`, { body, ...options });
68
- }
69
-
70
- /**
71
- * Update an existing system definition. Only the fields provided in the request
72
- * body will be updated. If a field is provided, the new content will replace the
73
- * existing content. If a field is not provided, the existing content will remain
74
- * unchanged.
75
- *
76
- * When updating schemas:
77
- *
78
- * - The system will accept your changes regardless of compatibility with existing
79
- * configurations
80
- * - Schema updates won't invalidate existing evaluations or configurations
81
- * - For significant redesigns, creating a new system definition provides a cleaner
82
- * separation
83
- *
84
- * @example
85
- * ```ts
86
- * const system = await client.systems.update(
87
- * '12345678-0a8b-4f66-b6f3-2ddcfa097257',
88
- * {
89
- * description:
90
- * 'Updated production chatbot powered by GPT-4 Turbo',
91
- * name: 'GPT-4 Turbo Chatbot',
92
- * },
93
- * );
94
- * ```
95
- */
96
- update(
97
- systemID: string,
98
- body: SystemUpdateParams | null | undefined = {},
99
- options?: RequestOptions,
100
- ): APIPromise<System> {
101
- return this._client.patch(path`/systems/${systemID}`, { body, ...options });
102
- }
103
-
104
- /**
105
- * Retrieve a paginated list of all systems. Systems are ordered by creation date.
106
- *
107
- * @example
108
- * ```ts
109
- * // Automatically fetches more pages as needed.
110
- * for await (const system of client.systems.list('314')) {
111
- * // ...
112
- * }
113
- * ```
114
- */
115
- list(
116
- projectID: string,
117
- query: SystemListParams | null | undefined = {},
118
- options?: RequestOptions,
119
- ): PagePromise<SystemsPaginatedResponse, System> {
120
- return this._client.getAPIList(path`/projects/${projectID}/systems`, PaginatedResponse<System>, {
121
- query,
122
- ...options,
123
- });
124
- }
125
-
126
- /**
127
- * Delete a system definition by ID. This will not delete associated system
128
- * configurations.
129
- *
130
- * @example
131
- * ```ts
132
- * const system = await client.systems.delete(
133
- * '12345678-0a8b-4f66-b6f3-2ddcfa097257',
134
- * );
135
- * ```
136
- */
137
- delete(systemID: string, options?: RequestOptions): APIPromise<SystemDeleteResponse> {
138
- return this._client.delete(path`/systems/${systemID}`, options);
139
- }
140
-
141
- /**
142
- * Retrieve a specific system by ID.
143
- *
144
- * @example
145
- * ```ts
146
- * const system = await client.systems.get(
147
- * '12345678-0a8b-4f66-b6f3-2ddcfa097257',
148
- * );
149
- * ```
150
- */
151
- get(systemID: string, options?: RequestOptions): APIPromise<System> {
152
- return this._client.get(path`/systems/${systemID}`, options);
153
- }
154
- }
155
-
156
- export type SystemsPaginatedResponse = PaginatedResponse<System>;
157
-
158
- /**
159
- * A System Under Test (SUT) defines the interface to a component or service you
160
- * want to evaluate.
161
- *
162
- * It specifies three contracts through schemas:
163
- *
164
- * - inputSchema: The structure of data the system accepts.
165
- * - outputSchema: The structure of data the system produces.
166
- * - configSchema: The parameters that modify system behavior.
167
- *
168
- * This abstraction lets you evaluate any system as a black box, focusing on its
169
- * interface rather than implementation details. It's particularly useful for
170
- * systems with variable outputs or complex internal state.
171
- *
172
- * Systems are templates - to run evaluations, pair them with a SystemConfig that
173
- * provides specific parameter values.
174
- */
175
- export interface System {
176
- /**
177
- * The ID of the system.
178
- */
179
- id: string;
180
-
181
- /**
182
- * The schema of the system's configuration.
183
- */
184
- configSchema: Record<string, unknown>;
185
-
186
- /**
187
- * The description of the system.
188
- */
189
- description: string;
190
-
191
- /**
192
- * The schema of the system's inputs.
193
- */
194
- inputSchema: Record<string, unknown>;
195
-
196
- /**
197
- * The name of the system.
198
- */
199
- name: string;
200
-
201
- /**
202
- * The schema of the system's outputs.
203
- */
204
- outputSchema: Record<string, unknown>;
205
- }
206
-
207
- export interface SystemDeleteResponse {
208
- /**
209
- * Whether the deletion was successful.
210
- */
211
- success: boolean;
212
- }
213
-
214
- export interface SystemCreateParams {
215
- /**
216
- * The schema of the system's configuration.
217
- */
218
- configSchema: Record<string, unknown>;
219
-
220
- /**
221
- * The description of the system.
222
- */
223
- description: string;
224
-
225
- /**
226
- * The schema of the system's inputs.
227
- */
228
- inputSchema: Record<string, unknown>;
229
-
230
- /**
231
- * The name of the system.
232
- */
233
- name: string;
234
-
235
- /**
236
- * The schema of the system's outputs.
237
- */
238
- outputSchema: Record<string, unknown>;
239
- }
240
-
241
- export interface SystemUpdateParams {
242
- /**
243
- * The schema of the system's configuration.
244
- */
245
- configSchema?: Record<string, unknown>;
246
-
247
- /**
248
- * The description of the system.
249
- */
250
- description?: string;
251
-
252
- /**
253
- * The schema of the system's inputs.
254
- */
255
- inputSchema?: Record<string, unknown>;
256
-
257
- /**
258
- * The name of the system.
259
- */
260
- name?: string;
261
-
262
- /**
263
- * The schema of the system's outputs.
264
- */
265
- outputSchema?: Record<string, unknown>;
266
- }
267
-
268
- export interface SystemListParams extends PaginatedResponseParams {}
269
-
270
- export declare namespace Systems {
271
- export {
272
- type System as System,
273
- type SystemDeleteResponse as SystemDeleteResponse,
274
- type SystemsPaginatedResponse as SystemsPaginatedResponse,
275
- type SystemCreateParams as SystemCreateParams,
276
- type SystemUpdateParams as SystemUpdateParams,
277
- type SystemListParams as SystemListParams,
278
- };
279
- }
3
+ export * from './systems/index';
@@ -102,7 +102,7 @@ export class Testsets extends APIResource {
102
102
  }
103
103
 
104
104
  /**
105
- * Get Testset by ID
105
+ * Get Testset
106
106
  *
107
107
  * @example
108
108
  * ```ts
package/src/version.ts CHANGED
@@ -1 +1 @@
1
- export const VERSION = '1.0.0-alpha.8'; // x-release-please-version
1
+ export const VERSION = '1.0.0'; // x-release-please-version
package/version.d.mts CHANGED
@@ -1,2 +1,2 @@
1
- export declare const VERSION = "1.0.0-alpha.8";
1
+ export declare const VERSION = "1.0.0";
2
2
  //# sourceMappingURL=version.d.mts.map
package/version.d.mts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"version.d.mts","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,kBAAkB,CAAC"}
1
+ {"version":3,"file":"version.d.mts","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC"}
package/version.d.ts CHANGED
@@ -1,2 +1,2 @@
1
- export declare const VERSION = "1.0.0-alpha.8";
1
+ export declare const VERSION = "1.0.0";
2
2
  //# sourceMappingURL=version.d.ts.map
package/version.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"version.d.ts","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,kBAAkB,CAAC"}
1
+ {"version":3,"file":"version.d.ts","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC"}
package/version.js CHANGED
@@ -1,5 +1,5 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.VERSION = void 0;
4
- exports.VERSION = '1.0.0-alpha.8'; // x-release-please-version
4
+ exports.VERSION = '1.0.0'; // x-release-please-version
5
5
  //# sourceMappingURL=version.js.map
package/version.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"version.js","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":";;;AAAa,QAAA,OAAO,GAAG,eAAe,CAAC,CAAC,2BAA2B"}
1
+ {"version":3,"file":"version.js","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":";;;AAAa,QAAA,OAAO,GAAG,OAAO,CAAC,CAAC,2BAA2B"}
package/version.mjs CHANGED
@@ -1,2 +1,2 @@
1
- export const VERSION = '1.0.0-alpha.8'; // x-release-please-version
1
+ export const VERSION = '1.0.0'; // x-release-please-version
2
2
  //# sourceMappingURL=version.mjs.map
package/version.mjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"version.mjs","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,eAAe,CAAC,CAAC,2BAA2B"}
1
+ {"version":3,"file":"version.mjs","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,2BAA2B"}
@@ -1,155 +0,0 @@
1
- import { APIResource } from "../core/resource.mjs";
2
- import { APIPromise } from "../core/api-promise.mjs";
3
- import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from "../core/pagination.mjs";
4
- import { RequestOptions } from "../internal/request-options.mjs";
5
- export declare class SystemConfigs extends APIResource {
6
- /**
7
- * Create a new configuration for a system.
8
- *
9
- * Each configuration contains specific parameter values that match the system's
10
- * configSchema - things like model parameters, thresholds, or processing options.
11
- * Once created, configurations cannot be modified, ensuring stable reference
12
- * points for evaluations.
13
- *
14
- * When creating a configuration:
15
- *
16
- * - The 'config' object is validated against the parent system's configSchema
17
- * - Configurations with validation errors are still stored, with errors included
18
- * in the response
19
- * - Validation errors indicate fields that don't match the schema but don't
20
- * prevent creation
21
- * - Having validation errors may affect how some evaluation metrics are calculated
22
- *
23
- * @example
24
- * ```ts
25
- * const systemConfig = await client.systemConfigs.create(
26
- * '12345678-0a8b-4f66-b6f3-2ddcfa097257',
27
- * {
28
- * config: {
29
- * temperature: 0.1,
30
- * maxTokens: 1024,
31
- * model: 'gpt-4-turbo',
32
- * },
33
- * name: 'Production (Low Temperature)',
34
- * },
35
- * );
36
- * ```
37
- */
38
- create(systemID: string, body: SystemConfigCreateParams, options?: RequestOptions): APIPromise<SystemConfig>;
39
- /**
40
- * Retrieve a paginated list of configurations for a specific system.
41
- *
42
- * System configurations provide concrete parameter values for a System Under Test,
43
- * defining exactly how the system should be configured during an evaluation run.
44
- *
45
- * @example
46
- * ```ts
47
- * // Automatically fetches more pages as needed.
48
- * for await (const systemConfig of client.systemConfigs.list(
49
- * '12345678-0a8b-4f66-b6f3-2ddcfa097257',
50
- * )) {
51
- * // ...
52
- * }
53
- * ```
54
- */
55
- list(systemID: string, query?: SystemConfigListParams | null | undefined, options?: RequestOptions): PagePromise<SystemConfigsPaginatedResponse, SystemConfig>;
56
- /**
57
- * Retrieve a specific system configuration by ID.
58
- *
59
- * @example
60
- * ```ts
61
- * const systemConfig = await client.systemConfigs.get(
62
- * '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
63
- * { systemId: '12345678-0a8b-4f66-b6f3-2ddcfa097257' },
64
- * );
65
- * ```
66
- */
67
- get(systemConfigID: string, params: SystemConfigGetParams, options?: RequestOptions): APIPromise<SystemConfig>;
68
- }
69
- export type SystemConfigsPaginatedResponse = PaginatedResponse<SystemConfig>;
70
- /**
71
- * A SystemConfig defines the specific settings for a System Under Test.
72
- *
73
- * Configurations contain parameter values that determine system behavior during
74
- * evaluation. They are immutable snapshots - once created, they never change.
75
- *
76
- * When running evaluations, you reference a specific configId to establish which
77
- * configuration to test.
78
- *
79
- * Configurations will be validated against the system's configSchema, with
80
- * non-conforming values generating warnings.
81
- */
82
- export interface SystemConfig {
83
- /**
84
- * The ID of the system configuration.
85
- */
86
- id: string;
87
- /**
88
- * The configuration of the system.
89
- */
90
- config: Record<string, unknown>;
91
- /**
92
- * The name of the system configuration.
93
- */
94
- name: string;
95
- /**
96
- * The ID of the system the configuration belongs to.
97
- */
98
- systemId: string;
99
- /**
100
- * Validation errors found in the configuration. If present, the configuration
101
- * doesn't fully conform to its system's configSchema.
102
- */
103
- validationErrors?: Array<SystemConfig.ValidationError>;
104
- }
105
- export declare namespace SystemConfig {
106
- interface ValidationError {
107
- /**
108
- * Human-readable error description.
109
- */
110
- message: string;
111
- /**
112
- * JSON Pointer to the field with the validation error.
113
- */
114
- path: string;
115
- }
116
- }
117
- export interface SystemConfigCreateParams {
118
- /**
119
- * The configuration of the system.
120
- */
121
- config: Record<string, unknown>;
122
- /**
123
- * The name of the system configuration.
124
- */
125
- name: string;
126
- /**
127
- * Validation errors found in the configuration. If present, the configuration
128
- * doesn't fully conform to its system's configSchema.
129
- */
130
- validationErrors?: Array<SystemConfigCreateParams.ValidationError>;
131
- }
132
- export declare namespace SystemConfigCreateParams {
133
- interface ValidationError {
134
- /**
135
- * Human-readable error description.
136
- */
137
- message: string;
138
- /**
139
- * JSON Pointer to the field with the validation error.
140
- */
141
- path: string;
142
- }
143
- }
144
- export interface SystemConfigListParams extends PaginatedResponseParams {
145
- }
146
- export interface SystemConfigGetParams {
147
- /**
148
- * The ID of the system the configuration belongs to.
149
- */
150
- systemId: string;
151
- }
152
- export declare namespace SystemConfigs {
153
- export { type SystemConfig as SystemConfig, type SystemConfigsPaginatedResponse as SystemConfigsPaginatedResponse, type SystemConfigCreateParams as SystemConfigCreateParams, type SystemConfigListParams as SystemConfigListParams, type SystemConfigGetParams as SystemConfigGetParams, };
154
- }
155
- //# sourceMappingURL=system-configs.d.mts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"system-configs.d.mts","sourceRoot":"","sources":["../src/resources/system-configs.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,WAAW,EAAE,iBAAiB,EAAE,KAAK,uBAAuB,EAAE;OAChE,EAAE,cAAc,EAAE;AAGzB,qBAAa,aAAc,SAAQ,WAAW;IAC5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACH,MAAM,CACJ,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,wBAAwB,EAC9B,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,YAAY,CAAC;IAI3B;;;;;;;;;;;;;;;OAeG;IACH,IAAI,CACF,QAAQ,EAAE,MAAM,EAChB,KAAK,GAAE,sBAAsB,GAAG,IAAI,GAAG,SAAc,EACrD,OAAO,CAAC,EAAE,cAAc,GACvB,WAAW,CAAC,8BAA8B,EAAE,YAAY,CAAC;IAO5D;;;;;;;;;;OAUG;IACH,GAAG,CACD,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,qBAAqB,EAC7B,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,YAAY,CAAC;CAI5B;AAED,MAAM,MAAM,8BAA8B,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;AAE7E;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEhC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;CACxD;AAED,yBAAiB,YAAY,CAAC;IAC5B,UAAiB,eAAe;QAC9B;;WAEG;QACH,OAAO,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;KACd;CACF;AAED,MAAM,WAAW,wBAAwB;IACvC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEhC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,gBAAgB,CAAC,EAAE,KAAK,CAAC,wBAAwB,CAAC,eAAe,CAAC,CAAC;CACpE;AAED,yBAAiB,wBAAwB,CAAC;IACxC,UAAiB,eAAe;QAC9B;;WAEG;QACH,OAAO,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;KACd;CACF;AAED,MAAM,WAAW,sBAAuB,SAAQ,uBAAuB;CAAG;AAE1E,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,CAAC,OAAO,WAAW,aAAa,CAAC;IACrC,OAAO,EACL,KAAK,YAAY,IAAI,YAAY,EACjC,KAAK,8BAA8B,IAAI,8BAA8B,EACrE,KAAK,wBAAwB,IAAI,wBAAwB,EACzD,KAAK,sBAAsB,IAAI,sBAAsB,EACrD,KAAK,qBAAqB,IAAI,qBAAqB,GACpD,CAAC;CACH"}
@@ -1,155 +0,0 @@
1
- import { APIResource } from "../core/resource.js";
2
- import { APIPromise } from "../core/api-promise.js";
3
- import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from "../core/pagination.js";
4
- import { RequestOptions } from "../internal/request-options.js";
5
- export declare class SystemConfigs extends APIResource {
6
- /**
7
- * Create a new configuration for a system.
8
- *
9
- * Each configuration contains specific parameter values that match the system's
10
- * configSchema - things like model parameters, thresholds, or processing options.
11
- * Once created, configurations cannot be modified, ensuring stable reference
12
- * points for evaluations.
13
- *
14
- * When creating a configuration:
15
- *
16
- * - The 'config' object is validated against the parent system's configSchema
17
- * - Configurations with validation errors are still stored, with errors included
18
- * in the response
19
- * - Validation errors indicate fields that don't match the schema but don't
20
- * prevent creation
21
- * - Having validation errors may affect how some evaluation metrics are calculated
22
- *
23
- * @example
24
- * ```ts
25
- * const systemConfig = await client.systemConfigs.create(
26
- * '12345678-0a8b-4f66-b6f3-2ddcfa097257',
27
- * {
28
- * config: {
29
- * temperature: 0.1,
30
- * maxTokens: 1024,
31
- * model: 'gpt-4-turbo',
32
- * },
33
- * name: 'Production (Low Temperature)',
34
- * },
35
- * );
36
- * ```
37
- */
38
- create(systemID: string, body: SystemConfigCreateParams, options?: RequestOptions): APIPromise<SystemConfig>;
39
- /**
40
- * Retrieve a paginated list of configurations for a specific system.
41
- *
42
- * System configurations provide concrete parameter values for a System Under Test,
43
- * defining exactly how the system should be configured during an evaluation run.
44
- *
45
- * @example
46
- * ```ts
47
- * // Automatically fetches more pages as needed.
48
- * for await (const systemConfig of client.systemConfigs.list(
49
- * '12345678-0a8b-4f66-b6f3-2ddcfa097257',
50
- * )) {
51
- * // ...
52
- * }
53
- * ```
54
- */
55
- list(systemID: string, query?: SystemConfigListParams | null | undefined, options?: RequestOptions): PagePromise<SystemConfigsPaginatedResponse, SystemConfig>;
56
- /**
57
- * Retrieve a specific system configuration by ID.
58
- *
59
- * @example
60
- * ```ts
61
- * const systemConfig = await client.systemConfigs.get(
62
- * '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
63
- * { systemId: '12345678-0a8b-4f66-b6f3-2ddcfa097257' },
64
- * );
65
- * ```
66
- */
67
- get(systemConfigID: string, params: SystemConfigGetParams, options?: RequestOptions): APIPromise<SystemConfig>;
68
- }
69
- export type SystemConfigsPaginatedResponse = PaginatedResponse<SystemConfig>;
70
- /**
71
- * A SystemConfig defines the specific settings for a System Under Test.
72
- *
73
- * Configurations contain parameter values that determine system behavior during
74
- * evaluation. They are immutable snapshots - once created, they never change.
75
- *
76
- * When running evaluations, you reference a specific configId to establish which
77
- * configuration to test.
78
- *
79
- * Configurations will be validated against the system's configSchema, with
80
- * non-conforming values generating warnings.
81
- */
82
- export interface SystemConfig {
83
- /**
84
- * The ID of the system configuration.
85
- */
86
- id: string;
87
- /**
88
- * The configuration of the system.
89
- */
90
- config: Record<string, unknown>;
91
- /**
92
- * The name of the system configuration.
93
- */
94
- name: string;
95
- /**
96
- * The ID of the system the configuration belongs to.
97
- */
98
- systemId: string;
99
- /**
100
- * Validation errors found in the configuration. If present, the configuration
101
- * doesn't fully conform to its system's configSchema.
102
- */
103
- validationErrors?: Array<SystemConfig.ValidationError>;
104
- }
105
- export declare namespace SystemConfig {
106
- interface ValidationError {
107
- /**
108
- * Human-readable error description.
109
- */
110
- message: string;
111
- /**
112
- * JSON Pointer to the field with the validation error.
113
- */
114
- path: string;
115
- }
116
- }
117
- export interface SystemConfigCreateParams {
118
- /**
119
- * The configuration of the system.
120
- */
121
- config: Record<string, unknown>;
122
- /**
123
- * The name of the system configuration.
124
- */
125
- name: string;
126
- /**
127
- * Validation errors found in the configuration. If present, the configuration
128
- * doesn't fully conform to its system's configSchema.
129
- */
130
- validationErrors?: Array<SystemConfigCreateParams.ValidationError>;
131
- }
132
- export declare namespace SystemConfigCreateParams {
133
- interface ValidationError {
134
- /**
135
- * Human-readable error description.
136
- */
137
- message: string;
138
- /**
139
- * JSON Pointer to the field with the validation error.
140
- */
141
- path: string;
142
- }
143
- }
144
- export interface SystemConfigListParams extends PaginatedResponseParams {
145
- }
146
- export interface SystemConfigGetParams {
147
- /**
148
- * The ID of the system the configuration belongs to.
149
- */
150
- systemId: string;
151
- }
152
- export declare namespace SystemConfigs {
153
- export { type SystemConfig as SystemConfig, type SystemConfigsPaginatedResponse as SystemConfigsPaginatedResponse, type SystemConfigCreateParams as SystemConfigCreateParams, type SystemConfigListParams as SystemConfigListParams, type SystemConfigGetParams as SystemConfigGetParams, };
154
- }
155
- //# sourceMappingURL=system-configs.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"system-configs.d.ts","sourceRoot":"","sources":["../src/resources/system-configs.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,WAAW,EAAE,iBAAiB,EAAE,KAAK,uBAAuB,EAAE;OAChE,EAAE,cAAc,EAAE;AAGzB,qBAAa,aAAc,SAAQ,WAAW;IAC5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACH,MAAM,CACJ,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,wBAAwB,EAC9B,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,YAAY,CAAC;IAI3B;;;;;;;;;;;;;;;OAeG;IACH,IAAI,CACF,QAAQ,EAAE,MAAM,EAChB,KAAK,GAAE,sBAAsB,GAAG,IAAI,GAAG,SAAc,EACrD,OAAO,CAAC,EAAE,cAAc,GACvB,WAAW,CAAC,8BAA8B,EAAE,YAAY,CAAC;IAO5D;;;;;;;;;;OAUG;IACH,GAAG,CACD,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,qBAAqB,EAC7B,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,YAAY,CAAC;CAI5B;AAED,MAAM,MAAM,8BAA8B,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;AAE7E;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEhC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;CACxD;AAED,yBAAiB,YAAY,CAAC;IAC5B,UAAiB,eAAe;QAC9B;;WAEG;QACH,OAAO,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;KACd;CACF;AAED,MAAM,WAAW,wBAAwB;IACvC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEhC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,gBAAgB,CAAC,EAAE,KAAK,CAAC,wBAAwB,CAAC,eAAe,CAAC,CAAC;CACpE;AAED,yBAAiB,wBAAwB,CAAC;IACxC,UAAiB,eAAe;QAC9B;;WAEG;QACH,OAAO,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;KACd;CACF;AAED,MAAM,WAAW,sBAAuB,SAAQ,uBAAuB;CAAG;AAE1E,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,CAAC,OAAO,WAAW,aAAa,CAAC;IACrC,OAAO,EACL,KAAK,YAAY,IAAI,YAAY,EACjC,KAAK,8BAA8B,IAAI,8BAA8B,EACrE,KAAK,wBAAwB,IAAI,wBAAwB,EACzD,KAAK,sBAAsB,IAAI,sBAAsB,EACrD,KAAK,qBAAqB,IAAI,qBAAqB,GACpD,CAAC;CACH"}