scorecard-ai 1.0.0-alpha.6 → 1.0.0-alpha.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -0
- package/README.md +15 -5
- package/client.d.mts +11 -4
- package/client.d.mts.map +1 -1
- package/client.d.ts +11 -4
- package/client.d.ts.map +1 -1
- package/client.js +20 -0
- package/client.js.map +1 -1
- package/client.mjs +20 -0
- package/client.mjs.map +1 -1
- package/internal/headers.js +1 -1
- package/internal/headers.js.map +1 -1
- package/internal/headers.mjs +1 -1
- package/internal/headers.mjs.map +1 -1
- package/internal/to-file.d.mts +0 -1
- package/internal/to-file.d.mts.map +1 -1
- package/internal/to-file.d.ts +0 -1
- package/internal/to-file.d.ts.map +1 -1
- package/internal/to-file.js +3 -2
- package/internal/to-file.js.map +1 -1
- package/internal/to-file.mjs +3 -2
- package/internal/to-file.mjs.map +1 -1
- package/internal/uploads.d.mts +1 -1
- package/internal/uploads.d.mts.map +1 -1
- package/internal/uploads.d.ts +1 -1
- package/internal/uploads.d.ts.map +1 -1
- package/internal/uploads.js +14 -4
- package/internal/uploads.js.map +1 -1
- package/internal/uploads.mjs +12 -3
- package/internal/uploads.mjs.map +1 -1
- package/internal/utils/uuid.d.mts +1 -1
- package/internal/utils/uuid.d.mts.map +1 -1
- package/internal/utils/uuid.d.ts +1 -1
- package/internal/utils/uuid.d.ts.map +1 -1
- package/internal/utils/uuid.js +1 -2
- package/internal/utils/uuid.js.map +1 -1
- package/internal/utils/uuid.mjs +1 -2
- package/internal/utils/uuid.mjs.map +1 -1
- package/lib/runAndEvaluate.d.mts.map +1 -1
- package/lib/runAndEvaluate.d.ts.map +1 -1
- package/lib/runAndEvaluate.js +1 -5
- package/lib/runAndEvaluate.js.map +1 -1
- package/lib/runAndEvaluate.mjs +1 -5
- package/lib/runAndEvaluate.mjs.map +1 -1
- package/package.json +1 -5
- package/resources/index.d.mts +3 -2
- package/resources/index.d.mts.map +1 -1
- package/resources/index.d.ts +3 -2
- package/resources/index.d.ts.map +1 -1
- package/resources/index.js +3 -1
- package/resources/index.js.map +1 -1
- package/resources/index.mjs +1 -0
- package/resources/index.mjs.map +1 -1
- package/resources/projects.d.mts +39 -4
- package/resources/projects.d.mts.map +1 -1
- package/resources/projects.d.ts +39 -4
- package/resources/projects.d.ts.map +1 -1
- package/resources/projects.js +23 -4
- package/resources/projects.js.map +1 -1
- package/resources/projects.mjs +23 -4
- package/resources/projects.mjs.map +1 -1
- package/resources/records.d.mts +20 -8
- package/resources/records.d.mts.map +1 -1
- package/resources/records.d.ts +20 -8
- package/resources/records.d.ts.map +1 -1
- package/resources/records.js +12 -0
- package/resources/records.js.map +1 -1
- package/resources/records.mjs +12 -0
- package/resources/records.mjs.map +1 -1
- package/resources/runs.d.mts +10 -21
- package/resources/runs.d.mts.map +1 -1
- package/resources/runs.d.ts +10 -21
- package/resources/runs.d.ts.map +1 -1
- package/resources/runs.js +9 -6
- package/resources/runs.js.map +1 -1
- package/resources/runs.mjs +9 -6
- package/resources/runs.mjs.map +1 -1
- package/resources/scores.d.mts +83 -0
- package/resources/scores.d.mts.map +1 -0
- package/resources/scores.d.ts +83 -0
- package/resources/scores.d.ts.map +1 -0
- package/resources/scores.js +35 -0
- package/resources/scores.js.map +1 -0
- package/resources/scores.mjs +31 -0
- package/resources/scores.mjs.map +1 -0
- package/resources/system-configs.d.mts +33 -0
- package/resources/system-configs.d.mts.map +1 -1
- package/resources/system-configs.d.ts +33 -0
- package/resources/system-configs.d.ts.map +1 -1
- package/resources/system-configs.js +33 -0
- package/resources/system-configs.js.map +1 -1
- package/resources/system-configs.mjs +33 -0
- package/resources/system-configs.mjs.map +1 -1
- package/resources/systems.d.mts +73 -0
- package/resources/systems.d.mts.map +1 -1
- package/resources/systems.d.ts +73 -0
- package/resources/systems.d.ts.map +1 -1
- package/resources/systems.js +73 -0
- package/resources/systems.js.map +1 -1
- package/resources/systems.mjs +73 -0
- package/resources/systems.mjs.map +1 -1
- package/resources/testcases.d.mts +72 -18
- package/resources/testcases.d.mts.map +1 -1
- package/resources/testcases.d.ts +72 -18
- package/resources/testcases.d.ts.map +1 -1
- package/resources/testcases.js +62 -0
- package/resources/testcases.js.map +1 -1
- package/resources/testcases.mjs +62 -0
- package/resources/testcases.mjs.map +1 -1
- package/resources/testsets.d.mts +78 -30
- package/resources/testsets.d.mts.map +1 -1
- package/resources/testsets.d.ts +78 -30
- package/resources/testsets.d.ts.map +1 -1
- package/resources/testsets.js +48 -0
- package/resources/testsets.js.map +1 -1
- package/resources/testsets.mjs +48 -0
- package/resources/testsets.mjs.map +1 -1
- package/src/client.ts +31 -12
- package/src/internal/headers.ts +1 -1
- package/src/internal/to-file.ts +4 -2
- package/src/internal/uploads.ts +16 -4
- package/src/internal/utils/uuid.ts +1 -3
- package/src/lib/runAndEvaluate.ts +1 -6
- package/src/resources/index.ts +5 -3
- package/src/resources/projects.ts +48 -9
- package/src/resources/records.ts +22 -10
- package/src/resources/runs.ts +10 -46
- package/src/resources/scores.ts +98 -0
- package/src/resources/system-configs.ts +33 -0
- package/src/resources/systems.ts +73 -0
- package/src/resources/testcases.ts +73 -19
- package/src/resources/testsets.ts +78 -30
- package/src/version.ts +1 -1
- package/version.d.mts +1 -1
- package/version.d.ts +1 -1
- package/version.js +1 -1
- package/version.mjs +1 -1
- package/internal/shims/crypto.d.mts +0 -13
- package/internal/shims/crypto.d.mts.map +0 -1
- package/internal/shims/crypto.d.ts +0 -13
- package/internal/shims/crypto.d.ts.map +0 -1
- package/internal/shims/crypto.js +0 -13
- package/internal/shims/crypto.js.map +0 -1
- package/internal/shims/crypto.mjs +0 -9
- package/internal/shims/crypto.mjs.map +0 -1
- package/internal/shims/file.d.mts +0 -22
- package/internal/shims/file.d.mts.map +0 -1
- package/internal/shims/file.d.ts +0 -22
- package/internal/shims/file.d.ts.map +0 -1
- package/internal/shims/file.js +0 -16
- package/internal/shims/file.js.map +0 -1
- package/internal/shims/file.mjs +0 -12
- package/internal/shims/file.mjs.map +0 -1
- package/internal/shims/getBuiltinModule.d.mts +0 -11
- package/internal/shims/getBuiltinModule.d.mts.map +0 -1
- package/internal/shims/getBuiltinModule.d.ts +0 -11
- package/internal/shims/getBuiltinModule.d.ts.map +0 -1
- package/internal/shims/getBuiltinModule.js +0 -71
- package/internal/shims/getBuiltinModule.js.map +0 -1
- package/internal/shims/getBuiltinModule.mjs +0 -67
- package/internal/shims/getBuiltinModule.mjs.map +0 -1
- package/internal/shims/nullGetBuiltinModule.d.mts +0 -2
- package/internal/shims/nullGetBuiltinModule.d.mts.map +0 -1
- package/internal/shims/nullGetBuiltinModule.d.ts +0 -2
- package/internal/shims/nullGetBuiltinModule.d.ts.map +0 -1
- package/internal/shims/nullGetBuiltinModule.js +0 -5
- package/internal/shims/nullGetBuiltinModule.js.map +0 -1
- package/internal/shims/nullGetBuiltinModule.mjs +0 -2
- package/internal/shims/nullGetBuiltinModule.mjs.map +0 -1
- package/src/internal/shims/crypto.ts +0 -18
- package/src/internal/shims/file.ts +0 -32
- package/src/internal/shims/getBuiltinModule.ts +0 -66
- package/src/internal/shims/nullGetBuiltinModule.ts +0 -1
|
@@ -23,6 +23,21 @@ export class SystemConfigs extends APIResource {
|
|
|
23
23
|
* - Validation errors indicate fields that don't match the schema but don't
|
|
24
24
|
* prevent creation
|
|
25
25
|
* - Having validation errors may affect how some evaluation metrics are calculated
|
|
26
|
+
*
|
|
27
|
+
* @example
|
|
28
|
+
* ```ts
|
|
29
|
+
* const systemConfig = await client.systemConfigs.create(
|
|
30
|
+
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
31
|
+
* {
|
|
32
|
+
* config: {
|
|
33
|
+
* temperature: 0.1,
|
|
34
|
+
* maxTokens: 1024,
|
|
35
|
+
* model: 'gpt-4-turbo',
|
|
36
|
+
* },
|
|
37
|
+
* name: 'Production (Low Temperature)',
|
|
38
|
+
* },
|
|
39
|
+
* );
|
|
40
|
+
* ```
|
|
26
41
|
*/
|
|
27
42
|
create(
|
|
28
43
|
systemID: string,
|
|
@@ -37,6 +52,16 @@ export class SystemConfigs extends APIResource {
|
|
|
37
52
|
*
|
|
38
53
|
* System configurations provide concrete parameter values for a System Under Test,
|
|
39
54
|
* defining exactly how the system should be configured during an evaluation run.
|
|
55
|
+
*
|
|
56
|
+
* @example
|
|
57
|
+
* ```ts
|
|
58
|
+
* // Automatically fetches more pages as needed.
|
|
59
|
+
* for await (const systemConfig of client.systemConfigs.list(
|
|
60
|
+
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
61
|
+
* )) {
|
|
62
|
+
* // ...
|
|
63
|
+
* }
|
|
64
|
+
* ```
|
|
40
65
|
*/
|
|
41
66
|
list(
|
|
42
67
|
systemID: string,
|
|
@@ -51,6 +76,14 @@ export class SystemConfigs extends APIResource {
|
|
|
51
76
|
|
|
52
77
|
/**
|
|
53
78
|
* Retrieve a specific system configuration by ID.
|
|
79
|
+
*
|
|
80
|
+
* @example
|
|
81
|
+
* ```ts
|
|
82
|
+
* const systemConfig = await client.systemConfigs.get(
|
|
83
|
+
* '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
|
|
84
|
+
* { systemId: '12345678-0a8b-4f66-b6f3-2ddcfa097257' },
|
|
85
|
+
* );
|
|
86
|
+
* ```
|
|
54
87
|
*/
|
|
55
88
|
get(
|
|
56
89
|
systemConfigID: string,
|
package/src/resources/systems.ts
CHANGED
|
@@ -23,6 +23,45 @@ export class Systems extends APIResource {
|
|
|
23
23
|
*
|
|
24
24
|
* This separation lets you evaluate any system as a black box, focusing on its
|
|
25
25
|
* interface rather than implementation details.
|
|
26
|
+
*
|
|
27
|
+
* @example
|
|
28
|
+
* ```ts
|
|
29
|
+
* const system = await client.systems.create('314', {
|
|
30
|
+
* configSchema: {
|
|
31
|
+
* type: 'object',
|
|
32
|
+
* properties: {
|
|
33
|
+
* temperature: { type: 'number' },
|
|
34
|
+
* maxTokens: { type: 'integer' },
|
|
35
|
+
* model: { type: 'string', enum: ['gpt-4', 'gpt-4-turbo'] },
|
|
36
|
+
* },
|
|
37
|
+
* required: ['model'],
|
|
38
|
+
* },
|
|
39
|
+
* description: 'Production chatbot powered by GPT-4',
|
|
40
|
+
* inputSchema: {
|
|
41
|
+
* type: 'object',
|
|
42
|
+
* properties: {
|
|
43
|
+
* messages: {
|
|
44
|
+
* type: 'array',
|
|
45
|
+
* items: {
|
|
46
|
+
* type: 'object',
|
|
47
|
+
* properties: {
|
|
48
|
+
* role: { type: 'string', enum: ['system', 'user', 'assistant'] },
|
|
49
|
+
* content: { type: 'string' },
|
|
50
|
+
* },
|
|
51
|
+
* required: ['role', 'content'],
|
|
52
|
+
* },
|
|
53
|
+
* },
|
|
54
|
+
* },
|
|
55
|
+
* required: ['messages'],
|
|
56
|
+
* },
|
|
57
|
+
* name: 'GPT-4 Chatbot',
|
|
58
|
+
* outputSchema: {
|
|
59
|
+
* type: 'object',
|
|
60
|
+
* properties: { response: { type: 'string' } },
|
|
61
|
+
* required: ['response'],
|
|
62
|
+
* },
|
|
63
|
+
* });
|
|
64
|
+
* ```
|
|
26
65
|
*/
|
|
27
66
|
create(projectID: string, body: SystemCreateParams, options?: RequestOptions): APIPromise<System> {
|
|
28
67
|
return this._client.post(path`/projects/${projectID}/systems`, { body, ...options });
|
|
@@ -41,6 +80,18 @@ export class Systems extends APIResource {
|
|
|
41
80
|
* - Schema updates won't invalidate existing evaluations or configurations
|
|
42
81
|
* - For significant redesigns, creating a new system definition provides a cleaner
|
|
43
82
|
* separation
|
|
83
|
+
*
|
|
84
|
+
* @example
|
|
85
|
+
* ```ts
|
|
86
|
+
* const system = await client.systems.update(
|
|
87
|
+
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
88
|
+
* {
|
|
89
|
+
* description:
|
|
90
|
+
* 'Updated production chatbot powered by GPT-4 Turbo',
|
|
91
|
+
* name: 'GPT-4 Turbo Chatbot',
|
|
92
|
+
* },
|
|
93
|
+
* );
|
|
94
|
+
* ```
|
|
44
95
|
*/
|
|
45
96
|
update(
|
|
46
97
|
systemID: string,
|
|
@@ -52,6 +103,14 @@ export class Systems extends APIResource {
|
|
|
52
103
|
|
|
53
104
|
/**
|
|
54
105
|
* Retrieve a paginated list of all systems. Systems are ordered by creation date.
|
|
106
|
+
*
|
|
107
|
+
* @example
|
|
108
|
+
* ```ts
|
|
109
|
+
* // Automatically fetches more pages as needed.
|
|
110
|
+
* for await (const system of client.systems.list('314')) {
|
|
111
|
+
* // ...
|
|
112
|
+
* }
|
|
113
|
+
* ```
|
|
55
114
|
*/
|
|
56
115
|
list(
|
|
57
116
|
projectID: string,
|
|
@@ -67,6 +126,13 @@ export class Systems extends APIResource {
|
|
|
67
126
|
/**
|
|
68
127
|
* Delete a system definition by ID. This will not delete associated system
|
|
69
128
|
* configurations.
|
|
129
|
+
*
|
|
130
|
+
* @example
|
|
131
|
+
* ```ts
|
|
132
|
+
* const system = await client.systems.delete(
|
|
133
|
+
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
134
|
+
* );
|
|
135
|
+
* ```
|
|
70
136
|
*/
|
|
71
137
|
delete(systemID: string, options?: RequestOptions): APIPromise<SystemDeleteResponse> {
|
|
72
138
|
return this._client.delete(path`/systems/${systemID}`, options);
|
|
@@ -74,6 +140,13 @@ export class Systems extends APIResource {
|
|
|
74
140
|
|
|
75
141
|
/**
|
|
76
142
|
* Retrieve a specific system by ID.
|
|
143
|
+
*
|
|
144
|
+
* @example
|
|
145
|
+
* ```ts
|
|
146
|
+
* const system = await client.systems.get(
|
|
147
|
+
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
148
|
+
* );
|
|
149
|
+
* ```
|
|
77
150
|
*/
|
|
78
151
|
get(systemID: string, options?: RequestOptions): APIPromise<System> {
|
|
79
152
|
return this._client.get(path`/systems/${systemID}`, options);
|
|
@@ -9,6 +9,37 @@ import { path } from '../internal/utils/path';
|
|
|
9
9
|
export class Testcases extends APIResource {
|
|
10
10
|
/**
|
|
11
11
|
* Create multiple Testcases in the specified Testset.
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* ```ts
|
|
15
|
+
* const testcase = await client.testcases.create('246', {
|
|
16
|
+
* items: [
|
|
17
|
+
* {
|
|
18
|
+
* jsonData: {
|
|
19
|
+
* question: 'What is the capital of France?',
|
|
20
|
+
* idealAnswer: 'Paris',
|
|
21
|
+
* provenance: 'hand_curated',
|
|
22
|
+
* },
|
|
23
|
+
* },
|
|
24
|
+
* {
|
|
25
|
+
* jsonData: {
|
|
26
|
+
* question:
|
|
27
|
+
* 'What is the largest planet in our solar system?',
|
|
28
|
+
* idealAnswer: 'Jupiter',
|
|
29
|
+
* provenance: 'synthetic',
|
|
30
|
+
* },
|
|
31
|
+
* },
|
|
32
|
+
* {
|
|
33
|
+
* jsonData: {
|
|
34
|
+
* question:
|
|
35
|
+
* 'How many planets are in our solar system?',
|
|
36
|
+
* idealAnswer: 8,
|
|
37
|
+
* provenance: 'user_feedback',
|
|
38
|
+
* },
|
|
39
|
+
* },
|
|
40
|
+
* ],
|
|
41
|
+
* });
|
|
42
|
+
* ```
|
|
12
43
|
*/
|
|
13
44
|
create(
|
|
14
45
|
testsetID: string,
|
|
@@ -20,6 +51,17 @@ export class Testcases extends APIResource {
|
|
|
20
51
|
|
|
21
52
|
/**
|
|
22
53
|
* Replace the data of an existing Testcase while keeping its ID.
|
|
54
|
+
*
|
|
55
|
+
* @example
|
|
56
|
+
* ```ts
|
|
57
|
+
* const testcase = await client.testcases.update('248', {
|
|
58
|
+
* jsonData: {
|
|
59
|
+
* question: 'What is the capital of France?',
|
|
60
|
+
* idealAnswer: 'Paris is the capital of France',
|
|
61
|
+
* provenance: 'hand_curated',
|
|
62
|
+
* },
|
|
63
|
+
* });
|
|
64
|
+
* ```
|
|
23
65
|
*/
|
|
24
66
|
update(testcaseID: string, body: TestcaseUpdateParams, options?: RequestOptions): APIPromise<Testcase> {
|
|
25
67
|
return this._client.put(path`/testcases/${testcaseID}`, { body, ...options });
|
|
@@ -27,6 +69,14 @@ export class Testcases extends APIResource {
|
|
|
27
69
|
|
|
28
70
|
/**
|
|
29
71
|
* Retrieve a paginated list of Testcases belonging to a Testset.
|
|
72
|
+
*
|
|
73
|
+
* @example
|
|
74
|
+
* ```ts
|
|
75
|
+
* // Automatically fetches more pages as needed.
|
|
76
|
+
* for await (const testcase of client.testcases.list('246')) {
|
|
77
|
+
* // ...
|
|
78
|
+
* }
|
|
79
|
+
* ```
|
|
30
80
|
*/
|
|
31
81
|
list(
|
|
32
82
|
testsetID: string,
|
|
@@ -41,6 +91,13 @@ export class Testcases extends APIResource {
|
|
|
41
91
|
|
|
42
92
|
/**
|
|
43
93
|
* Delete multiple Testcases by their IDs.
|
|
94
|
+
*
|
|
95
|
+
* @example
|
|
96
|
+
* ```ts
|
|
97
|
+
* const testcase = await client.testcases.delete({
|
|
98
|
+
* ids: ['123', '124', '125'],
|
|
99
|
+
* });
|
|
100
|
+
* ```
|
|
44
101
|
*/
|
|
45
102
|
delete(body: TestcaseDeleteParams, options?: RequestOptions): APIPromise<TestcaseDeleteResponse> {
|
|
46
103
|
return this._client.post('/testcases/bulk-delete', { body, ...options });
|
|
@@ -48,6 +105,11 @@ export class Testcases extends APIResource {
|
|
|
48
105
|
|
|
49
106
|
/**
|
|
50
107
|
* Retrieve a specific Testcase by ID.
|
|
108
|
+
*
|
|
109
|
+
* @example
|
|
110
|
+
* ```ts
|
|
111
|
+
* const testcase = await client.testcases.get('248');
|
|
112
|
+
* ```
|
|
51
113
|
*/
|
|
52
114
|
get(testcaseID: string, options?: RequestOptions): APIPromise<Testcase> {
|
|
53
115
|
return this._client.get(path`/testcases/${testcaseID}`, options);
|
|
@@ -58,11 +120,11 @@ export type TestcasesPaginatedResponse = PaginatedResponse<Testcase>;
|
|
|
58
120
|
|
|
59
121
|
/**
|
|
60
122
|
* A test case in the Scorecard system. Contains JSON data that is validated
|
|
61
|
-
* against the schema defined by its Testset. The `inputs` and `
|
|
62
|
-
* derived from the `data` field based on the Testset's `fieldMapping`, and
|
|
63
|
-
* all mapped fields, including those with validation errors. Testcases are
|
|
64
|
-
* regardless of validation results, with any validation errors included in
|
|
65
|
-
* `validationErrors` field.
|
|
123
|
+
* against the schema defined by its Testset. The `inputs` and `expected` fields
|
|
124
|
+
* are derived from the `data` field based on the Testset's `fieldMapping`, and
|
|
125
|
+
* include all mapped fields, including those with validation errors. Testcases are
|
|
126
|
+
* stored regardless of validation results, with any validation errors included in
|
|
127
|
+
* the `validationErrors` field.
|
|
66
128
|
*/
|
|
67
129
|
export interface Testcase {
|
|
68
130
|
/**
|
|
@@ -70,6 +132,12 @@ export interface Testcase {
|
|
|
70
132
|
*/
|
|
71
133
|
id: string;
|
|
72
134
|
|
|
135
|
+
/**
|
|
136
|
+
* Derived from data based on the Testset's fieldMapping. Contains all fields
|
|
137
|
+
* marked as expected outputs, including those with validation errors.
|
|
138
|
+
*/
|
|
139
|
+
expected: Record<string, unknown>;
|
|
140
|
+
|
|
73
141
|
/**
|
|
74
142
|
* Derived from data based on the Testset's fieldMapping. Contains all fields
|
|
75
143
|
* marked as inputs, including those with validation errors.
|
|
@@ -81,12 +149,6 @@ export interface Testcase {
|
|
|
81
149
|
*/
|
|
82
150
|
jsonData: Record<string, unknown>;
|
|
83
151
|
|
|
84
|
-
/**
|
|
85
|
-
* Derived from data based on the Testset's fieldMapping. Contains all fields
|
|
86
|
-
* marked as labels, including those with validation errors.
|
|
87
|
-
*/
|
|
88
|
-
labels: Record<string, unknown>;
|
|
89
|
-
|
|
90
152
|
/**
|
|
91
153
|
* The ID of the Testset this Testcase belongs to.
|
|
92
154
|
*/
|
|
@@ -132,14 +194,6 @@ export interface TestcaseCreateParams {
|
|
|
132
194
|
}
|
|
133
195
|
|
|
134
196
|
export namespace TestcaseCreateParams {
|
|
135
|
-
/**
|
|
136
|
-
* A test case in the Scorecard system. Contains JSON data that is validated
|
|
137
|
-
* against the schema defined by its Testset. The `inputs` and `labels` fields are
|
|
138
|
-
* derived from the `data` field based on the Testset's `fieldMapping`, and include
|
|
139
|
-
* all mapped fields, including those with validation errors. Testcases are stored
|
|
140
|
-
* regardless of validation results, with any validation errors included in the
|
|
141
|
-
* `validationErrors` field.
|
|
142
|
-
*/
|
|
143
197
|
export interface Item {
|
|
144
198
|
/**
|
|
145
199
|
* The JSON data of the Testcase, which is validated against the Testset's schema.
|
|
@@ -10,6 +10,28 @@ export class Testsets extends APIResource {
|
|
|
10
10
|
/**
|
|
11
11
|
* Create a new Testset for a Project. The Testset will be created in the Project
|
|
12
12
|
* specified in the path.
|
|
13
|
+
*
|
|
14
|
+
* @example
|
|
15
|
+
* ```ts
|
|
16
|
+
* const testset = await client.testsets.create('314', {
|
|
17
|
+
* description: 'Testset for long context Q&A chatbot.',
|
|
18
|
+
* fieldMapping: {
|
|
19
|
+
* inputs: ['question'],
|
|
20
|
+
* expected: ['idealAnswer'],
|
|
21
|
+
* metadata: [],
|
|
22
|
+
* },
|
|
23
|
+
* jsonSchema: {
|
|
24
|
+
* type: 'object',
|
|
25
|
+
* properties: {
|
|
26
|
+
* question: { type: 'string' },
|
|
27
|
+
* idealAnswer: { type: 'string' },
|
|
28
|
+
* provenance: { type: 'string' },
|
|
29
|
+
* geo: { type: 'string' },
|
|
30
|
+
* },
|
|
31
|
+
* },
|
|
32
|
+
* name: 'Long Context Q&A',
|
|
33
|
+
* });
|
|
34
|
+
* ```
|
|
13
35
|
*/
|
|
14
36
|
create(projectID: string, body: TestsetCreateParams, options?: RequestOptions): APIPromise<Testset> {
|
|
15
37
|
return this._client.post(path`/projects/${projectID}/testsets`, { body, ...options });
|
|
@@ -28,6 +50,14 @@ export class Testsets extends APIResource {
|
|
|
28
50
|
* updated schema
|
|
29
51
|
* - For complete control, provide both schema and fieldMapping when updating the
|
|
30
52
|
* schema
|
|
53
|
+
*
|
|
54
|
+
* @example
|
|
55
|
+
* ```ts
|
|
56
|
+
* const testset = await client.testsets.update('246', {
|
|
57
|
+
* description: 'Updated description for the Q&A Testset.',
|
|
58
|
+
* name: 'Updated Q&A Testset',
|
|
59
|
+
* });
|
|
60
|
+
* ```
|
|
31
61
|
*/
|
|
32
62
|
update(
|
|
33
63
|
testsetID: string,
|
|
@@ -39,6 +69,14 @@ export class Testsets extends APIResource {
|
|
|
39
69
|
|
|
40
70
|
/**
|
|
41
71
|
* Retrieve a paginated list of Testsets belonging to a Project.
|
|
72
|
+
*
|
|
73
|
+
* @example
|
|
74
|
+
* ```ts
|
|
75
|
+
* // Automatically fetches more pages as needed.
|
|
76
|
+
* for await (const testset of client.testsets.list('314')) {
|
|
77
|
+
* // ...
|
|
78
|
+
* }
|
|
79
|
+
* ```
|
|
42
80
|
*/
|
|
43
81
|
list(
|
|
44
82
|
projectID: string,
|
|
@@ -53,6 +91,11 @@ export class Testsets extends APIResource {
|
|
|
53
91
|
|
|
54
92
|
/**
|
|
55
93
|
* Delete Testset
|
|
94
|
+
*
|
|
95
|
+
* @example
|
|
96
|
+
* ```ts
|
|
97
|
+
* const testset = await client.testsets.delete('246');
|
|
98
|
+
* ```
|
|
56
99
|
*/
|
|
57
100
|
delete(testsetID: string, options?: RequestOptions): APIPromise<TestsetDeleteResponse> {
|
|
58
101
|
return this._client.delete(path`/testsets/${testsetID}`, options);
|
|
@@ -60,6 +103,11 @@ export class Testsets extends APIResource {
|
|
|
60
103
|
|
|
61
104
|
/**
|
|
62
105
|
* Get Testset by ID
|
|
106
|
+
*
|
|
107
|
+
* @example
|
|
108
|
+
* ```ts
|
|
109
|
+
* const testset = await client.testsets.get('246');
|
|
110
|
+
* ```
|
|
63
111
|
*/
|
|
64
112
|
get(testsetID: string, options?: RequestOptions): APIPromise<Testset> {
|
|
65
113
|
return this._client.get(path`/testsets/${testsetID}`, options);
|
|
@@ -71,9 +119,9 @@ export type TestsetsPaginatedResponse = PaginatedResponse<Testset>;
|
|
|
71
119
|
/**
|
|
72
120
|
* A collection of Testcases that share the same schema. Each Testset defines the
|
|
73
121
|
* structure of its Testcases through a JSON schema. The `fieldMapping` object maps
|
|
74
|
-
* top-level keys of the Testcase schema to their roles (input/
|
|
75
|
-
* mentioned in the `fieldMapping` during creation or update are treated
|
|
76
|
-
* metadata.
|
|
122
|
+
* top-level keys of the Testcase schema to their roles (input/expected output).
|
|
123
|
+
* Fields not mentioned in the `fieldMapping` during creation or update are treated
|
|
124
|
+
* as metadata.
|
|
77
125
|
*
|
|
78
126
|
* ## JSON Schema validation constraints supported:
|
|
79
127
|
*
|
|
@@ -106,8 +154,8 @@ export interface Testset {
|
|
|
106
154
|
description: string;
|
|
107
155
|
|
|
108
156
|
/**
|
|
109
|
-
* Maps top-level keys of the Testcase schema to their roles (input/
|
|
110
|
-
* Unmapped fields are treated as metadata.
|
|
157
|
+
* Maps top-level keys of the Testcase schema to their roles (input/expected
|
|
158
|
+
* output). Unmapped fields are treated as metadata.
|
|
111
159
|
*/
|
|
112
160
|
fieldMapping: Testset.FieldMapping;
|
|
113
161
|
|
|
@@ -124,22 +172,22 @@ export interface Testset {
|
|
|
124
172
|
|
|
125
173
|
export namespace Testset {
|
|
126
174
|
/**
|
|
127
|
-
* Maps top-level keys of the Testcase schema to their roles (input/
|
|
128
|
-
* Unmapped fields are treated as metadata.
|
|
175
|
+
* Maps top-level keys of the Testcase schema to their roles (input/expected
|
|
176
|
+
* output). Unmapped fields are treated as metadata.
|
|
129
177
|
*/
|
|
130
178
|
export interface FieldMapping {
|
|
131
179
|
/**
|
|
132
|
-
* Fields that represent
|
|
180
|
+
* Fields that represent expected outputs.
|
|
133
181
|
*/
|
|
134
|
-
|
|
182
|
+
expected: Array<string>;
|
|
135
183
|
|
|
136
184
|
/**
|
|
137
|
-
* Fields that represent
|
|
185
|
+
* Fields that represent inputs to the AI system.
|
|
138
186
|
*/
|
|
139
|
-
|
|
187
|
+
inputs: Array<string>;
|
|
140
188
|
|
|
141
189
|
/**
|
|
142
|
-
* Fields that are not inputs or
|
|
190
|
+
* Fields that are not inputs or expected outputs.
|
|
143
191
|
*/
|
|
144
192
|
metadata: Array<string>;
|
|
145
193
|
}
|
|
@@ -159,8 +207,8 @@ export interface TestsetCreateParams {
|
|
|
159
207
|
description: string;
|
|
160
208
|
|
|
161
209
|
/**
|
|
162
|
-
* Maps top-level keys of the Testcase schema to their roles (input/
|
|
163
|
-
* Unmapped fields are treated as metadata.
|
|
210
|
+
* Maps top-level keys of the Testcase schema to their roles (input/expected
|
|
211
|
+
* output). Unmapped fields are treated as metadata.
|
|
164
212
|
*/
|
|
165
213
|
fieldMapping: TestsetCreateParams.FieldMapping;
|
|
166
214
|
|
|
@@ -177,22 +225,22 @@ export interface TestsetCreateParams {
|
|
|
177
225
|
|
|
178
226
|
export namespace TestsetCreateParams {
|
|
179
227
|
/**
|
|
180
|
-
* Maps top-level keys of the Testcase schema to their roles (input/
|
|
181
|
-
* Unmapped fields are treated as metadata.
|
|
228
|
+
* Maps top-level keys of the Testcase schema to their roles (input/expected
|
|
229
|
+
* output). Unmapped fields are treated as metadata.
|
|
182
230
|
*/
|
|
183
231
|
export interface FieldMapping {
|
|
184
232
|
/**
|
|
185
|
-
* Fields that represent
|
|
233
|
+
* Fields that represent expected outputs.
|
|
186
234
|
*/
|
|
187
|
-
|
|
235
|
+
expected: Array<string>;
|
|
188
236
|
|
|
189
237
|
/**
|
|
190
|
-
* Fields that represent
|
|
238
|
+
* Fields that represent inputs to the AI system.
|
|
191
239
|
*/
|
|
192
|
-
|
|
240
|
+
inputs: Array<string>;
|
|
193
241
|
|
|
194
242
|
/**
|
|
195
|
-
* Fields that are not inputs or
|
|
243
|
+
* Fields that are not inputs or expected outputs.
|
|
196
244
|
*/
|
|
197
245
|
metadata: Array<string>;
|
|
198
246
|
}
|
|
@@ -205,8 +253,8 @@ export interface TestsetUpdateParams {
|
|
|
205
253
|
description?: string;
|
|
206
254
|
|
|
207
255
|
/**
|
|
208
|
-
* Maps top-level keys of the Testcase schema to their roles (input/
|
|
209
|
-
* Unmapped fields are treated as metadata.
|
|
256
|
+
* Maps top-level keys of the Testcase schema to their roles (input/expected
|
|
257
|
+
* output). Unmapped fields are treated as metadata.
|
|
210
258
|
*/
|
|
211
259
|
fieldMapping?: TestsetUpdateParams.FieldMapping;
|
|
212
260
|
|
|
@@ -223,22 +271,22 @@ export interface TestsetUpdateParams {
|
|
|
223
271
|
|
|
224
272
|
export namespace TestsetUpdateParams {
|
|
225
273
|
/**
|
|
226
|
-
* Maps top-level keys of the Testcase schema to their roles (input/
|
|
227
|
-
* Unmapped fields are treated as metadata.
|
|
274
|
+
* Maps top-level keys of the Testcase schema to their roles (input/expected
|
|
275
|
+
* output). Unmapped fields are treated as metadata.
|
|
228
276
|
*/
|
|
229
277
|
export interface FieldMapping {
|
|
230
278
|
/**
|
|
231
|
-
* Fields that represent
|
|
279
|
+
* Fields that represent expected outputs.
|
|
232
280
|
*/
|
|
233
|
-
|
|
281
|
+
expected: Array<string>;
|
|
234
282
|
|
|
235
283
|
/**
|
|
236
|
-
* Fields that represent
|
|
284
|
+
* Fields that represent inputs to the AI system.
|
|
237
285
|
*/
|
|
238
|
-
|
|
286
|
+
inputs: Array<string>;
|
|
239
287
|
|
|
240
288
|
/**
|
|
241
|
-
* Fields that are not inputs or
|
|
289
|
+
* Fields that are not inputs or expected outputs.
|
|
242
290
|
*/
|
|
243
291
|
metadata: Array<string>;
|
|
244
292
|
}
|
package/src/version.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const VERSION = '1.0.0-alpha.
|
|
1
|
+
export const VERSION = '1.0.0-alpha.8'; // x-release-please-version
|
package/version.d.mts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export declare const VERSION = "1.0.0-alpha.
|
|
1
|
+
export declare const VERSION = "1.0.0-alpha.8";
|
|
2
2
|
//# sourceMappingURL=version.d.mts.map
|
package/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export declare const VERSION = "1.0.0-alpha.
|
|
1
|
+
export declare const VERSION = "1.0.0-alpha.8";
|
|
2
2
|
//# sourceMappingURL=version.d.ts.map
|
package/version.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.VERSION = void 0;
|
|
4
|
-
exports.VERSION = '1.0.0-alpha.
|
|
4
|
+
exports.VERSION = '1.0.0-alpha.8'; // x-release-please-version
|
|
5
5
|
//# sourceMappingURL=version.js.map
|
package/version.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export const VERSION = '1.0.0-alpha.
|
|
1
|
+
export const VERSION = '1.0.0-alpha.8'; // x-release-please-version
|
|
2
2
|
//# sourceMappingURL=version.mjs.map
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
type Crypto = {
|
|
2
|
-
/** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/getRandomValues) */
|
|
3
|
-
getRandomValues<T extends ArrayBufferView | null>(array: T): T;
|
|
4
|
-
/**
|
|
5
|
-
* Available only in secure contexts.
|
|
6
|
-
*
|
|
7
|
-
* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/randomUUID)
|
|
8
|
-
*/
|
|
9
|
-
randomUUID?: () => string;
|
|
10
|
-
};
|
|
11
|
-
export declare let getCrypto: () => Crypto | undefined;
|
|
12
|
-
export {};
|
|
13
|
-
//# sourceMappingURL=crypto.d.mts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"crypto.d.mts","sourceRoot":"","sources":["../../src/internal/shims/crypto.ts"],"names":[],"mappings":"AAEA,KAAK,MAAM,GAAG;IACZ,yFAAyF;IACzF,eAAe,CAAC,CAAC,SAAS,eAAe,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC;IAC/D;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,MAAM,CAAC;CAC3B,CAAC;AACF,eAAO,IAAI,SAAS,EAAE,MAAM,MAAM,GAAG,SAKpC,CAAC"}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
type Crypto = {
|
|
2
|
-
/** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/getRandomValues) */
|
|
3
|
-
getRandomValues<T extends ArrayBufferView | null>(array: T): T;
|
|
4
|
-
/**
|
|
5
|
-
* Available only in secure contexts.
|
|
6
|
-
*
|
|
7
|
-
* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/randomUUID)
|
|
8
|
-
*/
|
|
9
|
-
randomUUID?: () => string;
|
|
10
|
-
};
|
|
11
|
-
export declare let getCrypto: () => Crypto | undefined;
|
|
12
|
-
export {};
|
|
13
|
-
//# sourceMappingURL=crypto.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"crypto.d.ts","sourceRoot":"","sources":["../../src/internal/shims/crypto.ts"],"names":[],"mappings":"AAEA,KAAK,MAAM,GAAG;IACZ,yFAAyF;IACzF,eAAe,CAAC,CAAC,SAAS,eAAe,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC;IAC/D;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,MAAM,CAAC;CAC3B,CAAC;AACF,eAAO,IAAI,SAAS,EAAE,MAAM,MAAM,GAAG,SAKpC,CAAC"}
|
package/internal/shims/crypto.js
DELETED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.getCrypto = void 0;
|
|
4
|
-
const getBuiltinModule_1 = require("./getBuiltinModule.js");
|
|
5
|
-
let getCrypto = function lazyGetCrypto() {
|
|
6
|
-
if (exports.getCrypto !== lazyGetCrypto)
|
|
7
|
-
return (0, exports.getCrypto)();
|
|
8
|
-
const crypto = globalThis.crypto || (0, getBuiltinModule_1.getBuiltinModule)?.('node:crypto')?.webcrypto;
|
|
9
|
-
exports.getCrypto = () => crypto;
|
|
10
|
-
return crypto;
|
|
11
|
-
};
|
|
12
|
-
exports.getCrypto = getCrypto;
|
|
13
|
-
//# sourceMappingURL=crypto.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"crypto.js","sourceRoot":"","sources":["../../src/internal/shims/crypto.ts"],"names":[],"mappings":";;;AAAA,4DAAsD;AAY/C,IAAI,SAAS,GAA6B,SAAS,aAAa;IACrE,IAAI,iBAAS,KAAK,aAAa;QAAE,OAAO,IAAA,iBAAS,GAAE,CAAC;IACpD,MAAM,MAAM,GAAY,UAAkB,CAAC,MAAM,IAAK,IAAA,mCAAgB,CAAA,EAAE,CAAC,aAAa,CAAS,EAAE,SAAS,CAAC;IAC3G,iBAAS,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AALS,QAAA,SAAS,aAKlB"}
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import { getBuiltinModule } from "./getBuiltinModule.mjs";
|
|
2
|
-
export let getCrypto = function lazyGetCrypto() {
|
|
3
|
-
if (getCrypto !== lazyGetCrypto)
|
|
4
|
-
return getCrypto();
|
|
5
|
-
const crypto = globalThis.crypto || getBuiltinModule?.('node:crypto')?.webcrypto;
|
|
6
|
-
getCrypto = () => crypto;
|
|
7
|
-
return crypto;
|
|
8
|
-
};
|
|
9
|
-
//# sourceMappingURL=crypto.mjs.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"crypto.mjs","sourceRoot":"","sources":["../../src/internal/shims/crypto.ts"],"names":[],"mappings":"OAAO,EAAE,gBAAgB,EAAE;AAY3B,MAAM,CAAC,IAAI,SAAS,GAA6B,SAAS,aAAa;IACrE,IAAI,SAAS,KAAK,aAAa;QAAE,OAAO,SAAS,EAAE,CAAC;IACpD,MAAM,MAAM,GAAY,UAAkB,CAAC,MAAM,IAAK,gBAAgB,EAAE,CAAC,aAAa,CAAS,EAAE,SAAS,CAAC;IAC3G,SAAS,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC"}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
export declare let getFile: () => FileConstructor;
|
|
2
|
-
type FileConstructor = typeof globalThis extends {
|
|
3
|
-
File: infer fileConstructor;
|
|
4
|
-
} ? fileConstructor : typeof FallbackFile;
|
|
5
|
-
export type File = InstanceType<FileConstructor>;
|
|
6
|
-
type FallbackBlobSource = ConstructorParameters<typeof Blob>[0] extends infer T ? T : never;
|
|
7
|
-
/**
|
|
8
|
-
* A [`File`](https://developer.mozilla.org/en-US/docs/Web/API/File) provides information about files.
|
|
9
|
-
*/
|
|
10
|
-
declare class FallbackFile extends Blob {
|
|
11
|
-
constructor(sources: FallbackBlobSource, fileName: string, options?: any);
|
|
12
|
-
/**
|
|
13
|
-
* The name of the `File`.
|
|
14
|
-
*/
|
|
15
|
-
readonly name: string;
|
|
16
|
-
/**
|
|
17
|
-
* The last modified date of the `File`.
|
|
18
|
-
*/
|
|
19
|
-
readonly lastModified: number;
|
|
20
|
-
}
|
|
21
|
-
export {};
|
|
22
|
-
//# sourceMappingURL=file.d.mts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"file.d.mts","sourceRoot":"","sources":["../../src/internal/shims/file.ts"],"names":[],"mappings":"AAEA,eAAO,IAAI,OAAO,QAA2B,eAO5C,CAAC;AAEF,KAAK,eAAe,GAClB,OAAO,UAAU,SAAS;IAAE,IAAI,EAAE,MAAM,eAAe,CAAA;CAAE,GAAG,eAAe,GAAG,OAAO,YAAY,CAAC;AACpG,MAAM,MAAM,IAAI,GAAG,YAAY,CAAC,eAAe,CAAC,CAAC;AAIjD,KAAK,kBAAkB,GAAG,qBAAqB,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;AAC5F;;GAEG;AACH,OAAO,OAAO,YAAa,SAAQ,IAAI;gBACzB,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG;IACxE;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B"}
|
package/internal/shims/file.d.ts
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
export declare let getFile: () => FileConstructor;
|
|
2
|
-
type FileConstructor = typeof globalThis extends {
|
|
3
|
-
File: infer fileConstructor;
|
|
4
|
-
} ? fileConstructor : typeof FallbackFile;
|
|
5
|
-
export type File = InstanceType<FileConstructor>;
|
|
6
|
-
type FallbackBlobSource = ConstructorParameters<typeof Blob>[0] extends infer T ? T : never;
|
|
7
|
-
/**
|
|
8
|
-
* A [`File`](https://developer.mozilla.org/en-US/docs/Web/API/File) provides information about files.
|
|
9
|
-
*/
|
|
10
|
-
declare class FallbackFile extends Blob {
|
|
11
|
-
constructor(sources: FallbackBlobSource, fileName: string, options?: any);
|
|
12
|
-
/**
|
|
13
|
-
* The name of the `File`.
|
|
14
|
-
*/
|
|
15
|
-
readonly name: string;
|
|
16
|
-
/**
|
|
17
|
-
* The last modified date of the `File`.
|
|
18
|
-
*/
|
|
19
|
-
readonly lastModified: number;
|
|
20
|
-
}
|
|
21
|
-
export {};
|
|
22
|
-
//# sourceMappingURL=file.d.ts.map
|