scorecard-ai 1.0.0-alpha.9 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/README.md +48 -19
- package/client.d.mts +4 -4
- package/client.d.mts.map +1 -1
- package/client.d.ts +4 -4
- package/client.d.ts.map +1 -1
- package/client.js +7 -5
- package/client.js.map +1 -1
- package/client.mjs +7 -5
- package/client.mjs.map +1 -1
- package/internal/tslib.js +6 -6
- package/lib/runAndEvaluate.d.mts +20 -8
- package/lib/runAndEvaluate.d.mts.map +1 -1
- package/lib/runAndEvaluate.d.ts +20 -8
- package/lib/runAndEvaluate.d.ts.map +1 -1
- package/lib/runAndEvaluate.js +31 -21
- package/lib/runAndEvaluate.js.map +1 -1
- package/lib/runAndEvaluate.mjs +31 -21
- package/lib/runAndEvaluate.mjs.map +1 -1
- package/package.json +1 -1
- package/resources/index.d.mts +2 -2
- package/resources/index.d.mts.map +1 -1
- package/resources/index.d.ts +2 -2
- package/resources/index.d.ts.map +1 -1
- package/resources/index.js +4 -4
- package/resources/index.js.map +1 -1
- package/resources/index.mjs +2 -2
- package/resources/index.mjs.map +1 -1
- package/resources/metrics.d.mts +611 -0
- package/resources/metrics.d.mts.map +1 -0
- package/resources/metrics.d.ts +611 -0
- package/resources/metrics.d.ts.map +1 -0
- package/resources/metrics.js +52 -0
- package/resources/metrics.js.map +1 -0
- package/resources/metrics.mjs +48 -0
- package/resources/metrics.mjs.map +1 -0
- package/resources/runs.d.mts +5 -5
- package/resources/runs.d.mts.map +1 -1
- package/resources/runs.d.ts +5 -5
- package/resources/runs.d.ts.map +1 -1
- package/resources/runs.js +1 -1
- package/resources/runs.mjs +1 -1
- package/resources/systems/index.d.mts +3 -0
- package/resources/systems/index.d.mts.map +1 -0
- package/resources/systems/index.d.ts +3 -0
- package/resources/systems/index.d.ts.map +1 -0
- package/resources/systems/index.js +9 -0
- package/resources/systems/index.js.map +1 -0
- package/resources/systems/index.mjs +4 -0
- package/resources/systems/index.mjs.map +1 -0
- package/resources/systems/systems.d.mts +229 -0
- package/resources/systems/systems.d.mts.map +1 -0
- package/resources/systems/systems.d.ts +229 -0
- package/resources/systems/systems.d.ts.map +1 -0
- package/resources/systems/systems.js +151 -0
- package/resources/systems/systems.js.map +1 -0
- package/resources/systems/systems.mjs +146 -0
- package/resources/systems/systems.mjs.map +1 -0
- package/resources/systems/versions.d.mts +132 -0
- package/resources/systems/versions.d.mts.map +1 -0
- package/resources/systems/versions.d.ts +132 -0
- package/resources/systems/versions.d.ts.map +1 -0
- package/resources/{system-configs.js → systems/versions.js} +26 -25
- package/resources/systems/versions.js.map +1 -0
- package/resources/{system-configs.mjs → systems/versions.mjs} +24 -23
- package/resources/systems/versions.mjs.map +1 -0
- package/resources/systems.d.mts +1 -224
- package/resources/systems.d.mts.map +1 -1
- package/resources/systems.d.ts +1 -224
- package/resources/systems.d.ts.map +1 -1
- package/resources/systems.js +2 -139
- package/resources/systems.js.map +1 -1
- package/resources/systems.mjs +1 -137
- package/resources/systems.mjs.map +1 -1
- package/resources/testsets.d.mts +1 -1
- package/resources/testsets.d.ts +1 -1
- package/resources/testsets.js +1 -1
- package/resources/testsets.mjs +1 -1
- package/src/client.ts +22 -27
- package/src/lib/runAndEvaluate.ts +52 -27
- package/src/resources/index.ts +2 -8
- package/src/resources/metrics.ts +768 -0
- package/src/resources/runs.ts +5 -5
- package/src/resources/systems/index.ts +18 -0
- package/src/resources/systems/systems.ts +299 -0
- package/src/resources/systems/versions.ts +166 -0
- package/src/resources/systems.ts +1 -277
- package/src/resources/testsets.ts +1 -1
- package/src/version.ts +1 -1
- package/version.d.mts +1 -1
- package/version.d.mts.map +1 -1
- package/version.d.ts +1 -1
- package/version.d.ts.map +1 -1
- package/version.js +1 -1
- package/version.js.map +1 -1
- package/version.mjs +1 -1
- package/version.mjs.map +1 -1
- package/resources/system-configs.d.mts +0 -148
- package/resources/system-configs.d.mts.map +0 -1
- package/resources/system-configs.d.ts +0 -148
- package/resources/system-configs.d.ts.map +0 -1
- package/resources/system-configs.js.map +0 -1
- package/resources/system-configs.mjs.map +0 -1
- package/src/resources/system-configs.ts +0 -189
|
@@ -1,28 +1,29 @@
|
|
|
1
1
|
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
2
|
-
import { APIResource } from "
|
|
3
|
-
import { PaginatedResponse } from "
|
|
4
|
-
import { path } from "
|
|
5
|
-
export class
|
|
2
|
+
import { APIResource } from "../../core/resource.mjs";
|
|
3
|
+
import { PaginatedResponse } from "../../core/pagination.mjs";
|
|
4
|
+
import { path } from "../../internal/utils/path.mjs";
|
|
5
|
+
export class Versions extends APIResource {
|
|
6
6
|
/**
|
|
7
|
-
* Create a new
|
|
7
|
+
* Create a new version for a system.
|
|
8
8
|
*
|
|
9
|
-
* Each
|
|
10
|
-
* configSchema - things like model parameters, thresholds, or processing
|
|
11
|
-
* Once created,
|
|
9
|
+
* Each version contains specific parameter values that match the system's
|
|
10
|
+
* `configSchema` - things like model parameters, thresholds, or processing
|
|
11
|
+
* options. Once created, versions cannot be modified, ensuring stable reference
|
|
12
12
|
* points for evaluations.
|
|
13
13
|
*
|
|
14
|
-
* When creating a
|
|
14
|
+
* When creating a system version:
|
|
15
15
|
*
|
|
16
|
-
* - The
|
|
17
|
-
* -
|
|
18
|
-
* in the response
|
|
16
|
+
* - The `config` object is validated against the parent system's `configSchema`.
|
|
17
|
+
* - System versions with validation errors are still stored, with errors included
|
|
18
|
+
* in the response.
|
|
19
19
|
* - Validation errors indicate fields that don't match the schema but don't
|
|
20
|
-
* prevent creation
|
|
21
|
-
* - Having validation errors may affect how some evaluation metrics are
|
|
20
|
+
* prevent creation.
|
|
21
|
+
* - Having validation errors may affect how some evaluation metrics are
|
|
22
|
+
* calculated.
|
|
22
23
|
*
|
|
23
24
|
* @example
|
|
24
25
|
* ```ts
|
|
25
|
-
* const
|
|
26
|
+
* const systemVersion = await client.systems.versions.create(
|
|
26
27
|
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
27
28
|
* {
|
|
28
29
|
* config: {
|
|
@@ -39,15 +40,15 @@ export class SystemConfigs extends APIResource {
|
|
|
39
40
|
return this._client.post(path `/systems/${systemID}/configs`, { body, ...options });
|
|
40
41
|
}
|
|
41
42
|
/**
|
|
42
|
-
* Retrieve a paginated list of
|
|
43
|
+
* Retrieve a paginated list of system versions for a specific system.
|
|
43
44
|
*
|
|
44
|
-
* System
|
|
45
|
+
* System versions provide concrete parameter values for a System Under Test,
|
|
45
46
|
* defining exactly how the system should be configured during an evaluation run.
|
|
46
47
|
*
|
|
47
48
|
* @example
|
|
48
49
|
* ```ts
|
|
49
50
|
* // Automatically fetches more pages as needed.
|
|
50
|
-
* for await (const
|
|
51
|
+
* for await (const systemVersion of client.systems.versions.list(
|
|
51
52
|
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
52
53
|
* )) {
|
|
53
54
|
* // ...
|
|
@@ -61,17 +62,17 @@ export class SystemConfigs extends APIResource {
|
|
|
61
62
|
});
|
|
62
63
|
}
|
|
63
64
|
/**
|
|
64
|
-
* Retrieve a specific system
|
|
65
|
+
* Retrieve a specific system version by ID.
|
|
65
66
|
*
|
|
66
67
|
* @example
|
|
67
68
|
* ```ts
|
|
68
|
-
* const
|
|
69
|
+
* const systemVersion = await client.systems.versions.get(
|
|
69
70
|
* '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
|
|
70
71
|
* );
|
|
71
72
|
* ```
|
|
72
73
|
*/
|
|
73
|
-
get(
|
|
74
|
-
return this._client.get(path `/systems/configs/${
|
|
74
|
+
get(systemVersionID, options) {
|
|
75
|
+
return this._client.get(path `/systems/configs/${systemVersionID}`, options);
|
|
75
76
|
}
|
|
76
77
|
}
|
|
77
|
-
//# sourceMappingURL=
|
|
78
|
+
//# sourceMappingURL=versions.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"versions.mjs","sourceRoot":"","sources":["../../src/resources/systems/versions.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;OAEf,EAAe,iBAAiB,EAAgC;OAEhE,EAAE,IAAI,EAAE;AAEf,MAAM,OAAO,QAAS,SAAQ,WAAW;IACvC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,MAAM,CAAC,QAAgB,EAAE,IAAyB,EAAE,OAAwB;QAC1E,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAA,YAAY,QAAQ,UAAU,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACrF,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,IAAI,CACF,QAAgB,EAChB,QAA8C,EAAE,EAChD,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAA,YAAY,QAAQ,UAAU,EAAE,CAAA,iBAAgC,CAAA,EAAE;YACnG,KAAK;YACL,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;OASG;IACH,GAAG,CAAC,eAAuB,EAAE,OAAwB;QACnD,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,oBAAoB,eAAe,EAAE,EAAE,OAAO,CAAC,CAAC;IAC9E,CAAC;CACF"}
|
package/resources/systems.d.mts
CHANGED
|
@@ -1,225 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
import { APIPromise } from "../core/api-promise.mjs";
|
|
3
|
-
import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from "../core/pagination.mjs";
|
|
4
|
-
import { RequestOptions } from "../internal/request-options.mjs";
|
|
5
|
-
export declare class Systems extends APIResource {
|
|
6
|
-
/**
|
|
7
|
-
* Create a new system definition that specifies the interface contracts for a
|
|
8
|
-
* component you want to evaluate.
|
|
9
|
-
*
|
|
10
|
-
* A system acts as a template that defines three key contracts through JSON
|
|
11
|
-
* Schemas:
|
|
12
|
-
*
|
|
13
|
-
* 1. Input Schema: What data your system accepts (e.g., user queries, context
|
|
14
|
-
* documents)
|
|
15
|
-
* 2. Output Schema: What data your system produces (e.g., responses, confidence
|
|
16
|
-
* scores)
|
|
17
|
-
* 3. Config Schema: What parameters can be adjusted (e.g., model selection,
|
|
18
|
-
* temperature)
|
|
19
|
-
*
|
|
20
|
-
* This separation lets you evaluate any system as a black box, focusing on its
|
|
21
|
-
* interface rather than implementation details.
|
|
22
|
-
*
|
|
23
|
-
* @example
|
|
24
|
-
* ```ts
|
|
25
|
-
* const system = await client.systems.create('314', {
|
|
26
|
-
* configSchema: {
|
|
27
|
-
* type: 'object',
|
|
28
|
-
* properties: {
|
|
29
|
-
* temperature: { type: 'number' },
|
|
30
|
-
* maxTokens: { type: 'integer' },
|
|
31
|
-
* model: { type: 'string', enum: ['gpt-4', 'gpt-4-turbo'] },
|
|
32
|
-
* },
|
|
33
|
-
* required: ['model'],
|
|
34
|
-
* },
|
|
35
|
-
* description: 'Production chatbot powered by GPT-4',
|
|
36
|
-
* inputSchema: {
|
|
37
|
-
* type: 'object',
|
|
38
|
-
* properties: {
|
|
39
|
-
* messages: {
|
|
40
|
-
* type: 'array',
|
|
41
|
-
* items: {
|
|
42
|
-
* type: 'object',
|
|
43
|
-
* properties: {
|
|
44
|
-
* role: { type: 'string', enum: ['system', 'user', 'assistant'] },
|
|
45
|
-
* content: { type: 'string' },
|
|
46
|
-
* },
|
|
47
|
-
* required: ['role', 'content'],
|
|
48
|
-
* },
|
|
49
|
-
* },
|
|
50
|
-
* },
|
|
51
|
-
* required: ['messages'],
|
|
52
|
-
* },
|
|
53
|
-
* name: 'GPT-4 Chatbot',
|
|
54
|
-
* outputSchema: {
|
|
55
|
-
* type: 'object',
|
|
56
|
-
* properties: { response: { type: 'string' } },
|
|
57
|
-
* required: ['response'],
|
|
58
|
-
* },
|
|
59
|
-
* });
|
|
60
|
-
* ```
|
|
61
|
-
*/
|
|
62
|
-
create(projectID: string, body: SystemCreateParams, options?: RequestOptions): APIPromise<System>;
|
|
63
|
-
/**
|
|
64
|
-
* Update an existing system definition. Only the fields provided in the request
|
|
65
|
-
* body will be updated. If a field is provided, the new content will replace the
|
|
66
|
-
* existing content. If a field is not provided, the existing content will remain
|
|
67
|
-
* unchanged.
|
|
68
|
-
*
|
|
69
|
-
* When updating schemas:
|
|
70
|
-
*
|
|
71
|
-
* - The system will accept your changes regardless of compatibility with existing
|
|
72
|
-
* configurations
|
|
73
|
-
* - Schema updates won't invalidate existing evaluations or configurations
|
|
74
|
-
* - For significant redesigns, creating a new system definition provides a cleaner
|
|
75
|
-
* separation
|
|
76
|
-
*
|
|
77
|
-
* @example
|
|
78
|
-
* ```ts
|
|
79
|
-
* const system = await client.systems.update(
|
|
80
|
-
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
81
|
-
* {
|
|
82
|
-
* description:
|
|
83
|
-
* 'Updated production chatbot powered by GPT-4 Turbo',
|
|
84
|
-
* name: 'GPT-4 Turbo Chatbot',
|
|
85
|
-
* },
|
|
86
|
-
* );
|
|
87
|
-
* ```
|
|
88
|
-
*/
|
|
89
|
-
update(systemID: string, body?: SystemUpdateParams | null | undefined, options?: RequestOptions): APIPromise<System>;
|
|
90
|
-
/**
|
|
91
|
-
* Retrieve a paginated list of all systems. Systems are ordered by creation date.
|
|
92
|
-
*
|
|
93
|
-
* @example
|
|
94
|
-
* ```ts
|
|
95
|
-
* // Automatically fetches more pages as needed.
|
|
96
|
-
* for await (const system of client.systems.list('314')) {
|
|
97
|
-
* // ...
|
|
98
|
-
* }
|
|
99
|
-
* ```
|
|
100
|
-
*/
|
|
101
|
-
list(projectID: string, query?: SystemListParams | null | undefined, options?: RequestOptions): PagePromise<SystemsPaginatedResponse, System>;
|
|
102
|
-
/**
|
|
103
|
-
* Delete a system definition by ID. This will not delete associated system
|
|
104
|
-
* configurations.
|
|
105
|
-
*
|
|
106
|
-
* @example
|
|
107
|
-
* ```ts
|
|
108
|
-
* const system = await client.systems.delete(
|
|
109
|
-
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
110
|
-
* );
|
|
111
|
-
* ```
|
|
112
|
-
*/
|
|
113
|
-
delete(systemID: string, options?: RequestOptions): APIPromise<SystemDeleteResponse>;
|
|
114
|
-
/**
|
|
115
|
-
* Retrieve a specific system by ID.
|
|
116
|
-
*
|
|
117
|
-
* @example
|
|
118
|
-
* ```ts
|
|
119
|
-
* const system = await client.systems.get(
|
|
120
|
-
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
121
|
-
* );
|
|
122
|
-
* ```
|
|
123
|
-
*/
|
|
124
|
-
get(systemID: string, options?: RequestOptions): APIPromise<System>;
|
|
125
|
-
}
|
|
126
|
-
export type SystemsPaginatedResponse = PaginatedResponse<System>;
|
|
127
|
-
/**
|
|
128
|
-
* A System Under Test (SUT) defines the interface to a component or service you
|
|
129
|
-
* want to evaluate.
|
|
130
|
-
*
|
|
131
|
-
* It specifies three contracts through schemas:
|
|
132
|
-
*
|
|
133
|
-
* - inputSchema: The structure of data the system accepts.
|
|
134
|
-
* - outputSchema: The structure of data the system produces.
|
|
135
|
-
* - configSchema: The parameters that modify system behavior.
|
|
136
|
-
*
|
|
137
|
-
* This abstraction lets you evaluate any system as a black box, focusing on its
|
|
138
|
-
* interface rather than implementation details. It's particularly useful for
|
|
139
|
-
* systems with variable outputs or complex internal state.
|
|
140
|
-
*
|
|
141
|
-
* Systems are templates - to run evaluations, pair them with a SystemConfig that
|
|
142
|
-
* provides specific parameter values.
|
|
143
|
-
*/
|
|
144
|
-
export interface System {
|
|
145
|
-
/**
|
|
146
|
-
* The ID of the system.
|
|
147
|
-
*/
|
|
148
|
-
id: string;
|
|
149
|
-
/**
|
|
150
|
-
* The schema of the system's configuration.
|
|
151
|
-
*/
|
|
152
|
-
configSchema: Record<string, unknown>;
|
|
153
|
-
/**
|
|
154
|
-
* The description of the system.
|
|
155
|
-
*/
|
|
156
|
-
description: string;
|
|
157
|
-
/**
|
|
158
|
-
* The schema of the system's inputs.
|
|
159
|
-
*/
|
|
160
|
-
inputSchema: Record<string, unknown>;
|
|
161
|
-
/**
|
|
162
|
-
* The name of the system.
|
|
163
|
-
*/
|
|
164
|
-
name: string;
|
|
165
|
-
/**
|
|
166
|
-
* The schema of the system's outputs.
|
|
167
|
-
*/
|
|
168
|
-
outputSchema: Record<string, unknown>;
|
|
169
|
-
}
|
|
170
|
-
export interface SystemDeleteResponse {
|
|
171
|
-
/**
|
|
172
|
-
* Whether the deletion was successful.
|
|
173
|
-
*/
|
|
174
|
-
success: boolean;
|
|
175
|
-
}
|
|
176
|
-
export interface SystemCreateParams {
|
|
177
|
-
/**
|
|
178
|
-
* The schema of the system's configuration.
|
|
179
|
-
*/
|
|
180
|
-
configSchema: Record<string, unknown>;
|
|
181
|
-
/**
|
|
182
|
-
* The description of the system.
|
|
183
|
-
*/
|
|
184
|
-
description: string;
|
|
185
|
-
/**
|
|
186
|
-
* The schema of the system's inputs.
|
|
187
|
-
*/
|
|
188
|
-
inputSchema: Record<string, unknown>;
|
|
189
|
-
/**
|
|
190
|
-
* The name of the system.
|
|
191
|
-
*/
|
|
192
|
-
name: string;
|
|
193
|
-
/**
|
|
194
|
-
* The schema of the system's outputs.
|
|
195
|
-
*/
|
|
196
|
-
outputSchema: Record<string, unknown>;
|
|
197
|
-
}
|
|
198
|
-
export interface SystemUpdateParams {
|
|
199
|
-
/**
|
|
200
|
-
* The schema of the system's configuration.
|
|
201
|
-
*/
|
|
202
|
-
configSchema?: Record<string, unknown>;
|
|
203
|
-
/**
|
|
204
|
-
* The description of the system.
|
|
205
|
-
*/
|
|
206
|
-
description?: string;
|
|
207
|
-
/**
|
|
208
|
-
* The schema of the system's inputs.
|
|
209
|
-
*/
|
|
210
|
-
inputSchema?: Record<string, unknown>;
|
|
211
|
-
/**
|
|
212
|
-
* The name of the system.
|
|
213
|
-
*/
|
|
214
|
-
name?: string;
|
|
215
|
-
/**
|
|
216
|
-
* The schema of the system's outputs.
|
|
217
|
-
*/
|
|
218
|
-
outputSchema?: Record<string, unknown>;
|
|
219
|
-
}
|
|
220
|
-
export interface SystemListParams extends PaginatedResponseParams {
|
|
221
|
-
}
|
|
222
|
-
export declare namespace Systems {
|
|
223
|
-
export { type System as System, type SystemDeleteResponse as SystemDeleteResponse, type SystemsPaginatedResponse as SystemsPaginatedResponse, type SystemCreateParams as SystemCreateParams, type SystemUpdateParams as SystemUpdateParams, type SystemListParams as SystemListParams, };
|
|
224
|
-
}
|
|
1
|
+
export * from "./systems/index.mjs";
|
|
225
2
|
//# sourceMappingURL=systems.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"systems.d.mts","sourceRoot":"","sources":["../src/resources/systems.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"systems.d.mts","sourceRoot":"","sources":["../src/resources/systems.ts"],"names":[],"mappings":""}
|
package/resources/systems.d.ts
CHANGED
|
@@ -1,225 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
import { APIPromise } from "../core/api-promise.js";
|
|
3
|
-
import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from "../core/pagination.js";
|
|
4
|
-
import { RequestOptions } from "../internal/request-options.js";
|
|
5
|
-
export declare class Systems extends APIResource {
|
|
6
|
-
/**
|
|
7
|
-
* Create a new system definition that specifies the interface contracts for a
|
|
8
|
-
* component you want to evaluate.
|
|
9
|
-
*
|
|
10
|
-
* A system acts as a template that defines three key contracts through JSON
|
|
11
|
-
* Schemas:
|
|
12
|
-
*
|
|
13
|
-
* 1. Input Schema: What data your system accepts (e.g., user queries, context
|
|
14
|
-
* documents)
|
|
15
|
-
* 2. Output Schema: What data your system produces (e.g., responses, confidence
|
|
16
|
-
* scores)
|
|
17
|
-
* 3. Config Schema: What parameters can be adjusted (e.g., model selection,
|
|
18
|
-
* temperature)
|
|
19
|
-
*
|
|
20
|
-
* This separation lets you evaluate any system as a black box, focusing on its
|
|
21
|
-
* interface rather than implementation details.
|
|
22
|
-
*
|
|
23
|
-
* @example
|
|
24
|
-
* ```ts
|
|
25
|
-
* const system = await client.systems.create('314', {
|
|
26
|
-
* configSchema: {
|
|
27
|
-
* type: 'object',
|
|
28
|
-
* properties: {
|
|
29
|
-
* temperature: { type: 'number' },
|
|
30
|
-
* maxTokens: { type: 'integer' },
|
|
31
|
-
* model: { type: 'string', enum: ['gpt-4', 'gpt-4-turbo'] },
|
|
32
|
-
* },
|
|
33
|
-
* required: ['model'],
|
|
34
|
-
* },
|
|
35
|
-
* description: 'Production chatbot powered by GPT-4',
|
|
36
|
-
* inputSchema: {
|
|
37
|
-
* type: 'object',
|
|
38
|
-
* properties: {
|
|
39
|
-
* messages: {
|
|
40
|
-
* type: 'array',
|
|
41
|
-
* items: {
|
|
42
|
-
* type: 'object',
|
|
43
|
-
* properties: {
|
|
44
|
-
* role: { type: 'string', enum: ['system', 'user', 'assistant'] },
|
|
45
|
-
* content: { type: 'string' },
|
|
46
|
-
* },
|
|
47
|
-
* required: ['role', 'content'],
|
|
48
|
-
* },
|
|
49
|
-
* },
|
|
50
|
-
* },
|
|
51
|
-
* required: ['messages'],
|
|
52
|
-
* },
|
|
53
|
-
* name: 'GPT-4 Chatbot',
|
|
54
|
-
* outputSchema: {
|
|
55
|
-
* type: 'object',
|
|
56
|
-
* properties: { response: { type: 'string' } },
|
|
57
|
-
* required: ['response'],
|
|
58
|
-
* },
|
|
59
|
-
* });
|
|
60
|
-
* ```
|
|
61
|
-
*/
|
|
62
|
-
create(projectID: string, body: SystemCreateParams, options?: RequestOptions): APIPromise<System>;
|
|
63
|
-
/**
|
|
64
|
-
* Update an existing system definition. Only the fields provided in the request
|
|
65
|
-
* body will be updated. If a field is provided, the new content will replace the
|
|
66
|
-
* existing content. If a field is not provided, the existing content will remain
|
|
67
|
-
* unchanged.
|
|
68
|
-
*
|
|
69
|
-
* When updating schemas:
|
|
70
|
-
*
|
|
71
|
-
* - The system will accept your changes regardless of compatibility with existing
|
|
72
|
-
* configurations
|
|
73
|
-
* - Schema updates won't invalidate existing evaluations or configurations
|
|
74
|
-
* - For significant redesigns, creating a new system definition provides a cleaner
|
|
75
|
-
* separation
|
|
76
|
-
*
|
|
77
|
-
* @example
|
|
78
|
-
* ```ts
|
|
79
|
-
* const system = await client.systems.update(
|
|
80
|
-
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
81
|
-
* {
|
|
82
|
-
* description:
|
|
83
|
-
* 'Updated production chatbot powered by GPT-4 Turbo',
|
|
84
|
-
* name: 'GPT-4 Turbo Chatbot',
|
|
85
|
-
* },
|
|
86
|
-
* );
|
|
87
|
-
* ```
|
|
88
|
-
*/
|
|
89
|
-
update(systemID: string, body?: SystemUpdateParams | null | undefined, options?: RequestOptions): APIPromise<System>;
|
|
90
|
-
/**
|
|
91
|
-
* Retrieve a paginated list of all systems. Systems are ordered by creation date.
|
|
92
|
-
*
|
|
93
|
-
* @example
|
|
94
|
-
* ```ts
|
|
95
|
-
* // Automatically fetches more pages as needed.
|
|
96
|
-
* for await (const system of client.systems.list('314')) {
|
|
97
|
-
* // ...
|
|
98
|
-
* }
|
|
99
|
-
* ```
|
|
100
|
-
*/
|
|
101
|
-
list(projectID: string, query?: SystemListParams | null | undefined, options?: RequestOptions): PagePromise<SystemsPaginatedResponse, System>;
|
|
102
|
-
/**
|
|
103
|
-
* Delete a system definition by ID. This will not delete associated system
|
|
104
|
-
* configurations.
|
|
105
|
-
*
|
|
106
|
-
* @example
|
|
107
|
-
* ```ts
|
|
108
|
-
* const system = await client.systems.delete(
|
|
109
|
-
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
110
|
-
* );
|
|
111
|
-
* ```
|
|
112
|
-
*/
|
|
113
|
-
delete(systemID: string, options?: RequestOptions): APIPromise<SystemDeleteResponse>;
|
|
114
|
-
/**
|
|
115
|
-
* Retrieve a specific system by ID.
|
|
116
|
-
*
|
|
117
|
-
* @example
|
|
118
|
-
* ```ts
|
|
119
|
-
* const system = await client.systems.get(
|
|
120
|
-
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
121
|
-
* );
|
|
122
|
-
* ```
|
|
123
|
-
*/
|
|
124
|
-
get(systemID: string, options?: RequestOptions): APIPromise<System>;
|
|
125
|
-
}
|
|
126
|
-
export type SystemsPaginatedResponse = PaginatedResponse<System>;
|
|
127
|
-
/**
|
|
128
|
-
* A System Under Test (SUT) defines the interface to a component or service you
|
|
129
|
-
* want to evaluate.
|
|
130
|
-
*
|
|
131
|
-
* It specifies three contracts through schemas:
|
|
132
|
-
*
|
|
133
|
-
* - inputSchema: The structure of data the system accepts.
|
|
134
|
-
* - outputSchema: The structure of data the system produces.
|
|
135
|
-
* - configSchema: The parameters that modify system behavior.
|
|
136
|
-
*
|
|
137
|
-
* This abstraction lets you evaluate any system as a black box, focusing on its
|
|
138
|
-
* interface rather than implementation details. It's particularly useful for
|
|
139
|
-
* systems with variable outputs or complex internal state.
|
|
140
|
-
*
|
|
141
|
-
* Systems are templates - to run evaluations, pair them with a SystemConfig that
|
|
142
|
-
* provides specific parameter values.
|
|
143
|
-
*/
|
|
144
|
-
export interface System {
|
|
145
|
-
/**
|
|
146
|
-
* The ID of the system.
|
|
147
|
-
*/
|
|
148
|
-
id: string;
|
|
149
|
-
/**
|
|
150
|
-
* The schema of the system's configuration.
|
|
151
|
-
*/
|
|
152
|
-
configSchema: Record<string, unknown>;
|
|
153
|
-
/**
|
|
154
|
-
* The description of the system.
|
|
155
|
-
*/
|
|
156
|
-
description: string;
|
|
157
|
-
/**
|
|
158
|
-
* The schema of the system's inputs.
|
|
159
|
-
*/
|
|
160
|
-
inputSchema: Record<string, unknown>;
|
|
161
|
-
/**
|
|
162
|
-
* The name of the system.
|
|
163
|
-
*/
|
|
164
|
-
name: string;
|
|
165
|
-
/**
|
|
166
|
-
* The schema of the system's outputs.
|
|
167
|
-
*/
|
|
168
|
-
outputSchema: Record<string, unknown>;
|
|
169
|
-
}
|
|
170
|
-
export interface SystemDeleteResponse {
|
|
171
|
-
/**
|
|
172
|
-
* Whether the deletion was successful.
|
|
173
|
-
*/
|
|
174
|
-
success: boolean;
|
|
175
|
-
}
|
|
176
|
-
export interface SystemCreateParams {
|
|
177
|
-
/**
|
|
178
|
-
* The schema of the system's configuration.
|
|
179
|
-
*/
|
|
180
|
-
configSchema: Record<string, unknown>;
|
|
181
|
-
/**
|
|
182
|
-
* The description of the system.
|
|
183
|
-
*/
|
|
184
|
-
description: string;
|
|
185
|
-
/**
|
|
186
|
-
* The schema of the system's inputs.
|
|
187
|
-
*/
|
|
188
|
-
inputSchema: Record<string, unknown>;
|
|
189
|
-
/**
|
|
190
|
-
* The name of the system.
|
|
191
|
-
*/
|
|
192
|
-
name: string;
|
|
193
|
-
/**
|
|
194
|
-
* The schema of the system's outputs.
|
|
195
|
-
*/
|
|
196
|
-
outputSchema: Record<string, unknown>;
|
|
197
|
-
}
|
|
198
|
-
export interface SystemUpdateParams {
|
|
199
|
-
/**
|
|
200
|
-
* The schema of the system's configuration.
|
|
201
|
-
*/
|
|
202
|
-
configSchema?: Record<string, unknown>;
|
|
203
|
-
/**
|
|
204
|
-
* The description of the system.
|
|
205
|
-
*/
|
|
206
|
-
description?: string;
|
|
207
|
-
/**
|
|
208
|
-
* The schema of the system's inputs.
|
|
209
|
-
*/
|
|
210
|
-
inputSchema?: Record<string, unknown>;
|
|
211
|
-
/**
|
|
212
|
-
* The name of the system.
|
|
213
|
-
*/
|
|
214
|
-
name?: string;
|
|
215
|
-
/**
|
|
216
|
-
* The schema of the system's outputs.
|
|
217
|
-
*/
|
|
218
|
-
outputSchema?: Record<string, unknown>;
|
|
219
|
-
}
|
|
220
|
-
export interface SystemListParams extends PaginatedResponseParams {
|
|
221
|
-
}
|
|
222
|
-
export declare namespace Systems {
|
|
223
|
-
export { type System as System, type SystemDeleteResponse as SystemDeleteResponse, type SystemsPaginatedResponse as SystemsPaginatedResponse, type SystemCreateParams as SystemCreateParams, type SystemUpdateParams as SystemUpdateParams, type SystemListParams as SystemListParams, };
|
|
224
|
-
}
|
|
1
|
+
export * from "./systems/index.js";
|
|
225
2
|
//# sourceMappingURL=systems.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"systems.d.ts","sourceRoot":"","sources":["../src/resources/systems.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"systems.d.ts","sourceRoot":"","sources":["../src/resources/systems.ts"],"names":[],"mappings":""}
|