scorecard-ai 1.0.0-alpha.5 → 1.0.0-alpha.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/README.md +17 -7
- package/client.d.mts +13 -6
- package/client.d.mts.map +1 -1
- package/client.d.ts +13 -6
- package/client.d.ts.map +1 -1
- package/client.js +27 -7
- package/client.js.map +1 -1
- package/client.mjs +27 -7
- package/client.mjs.map +1 -1
- package/internal/headers.js +1 -1
- package/internal/headers.js.map +1 -1
- package/internal/headers.mjs +1 -1
- package/internal/headers.mjs.map +1 -1
- package/internal/to-file.d.mts +0 -1
- package/internal/to-file.d.mts.map +1 -1
- package/internal/to-file.d.ts +0 -1
- package/internal/to-file.d.ts.map +1 -1
- package/internal/to-file.js +3 -2
- package/internal/to-file.js.map +1 -1
- package/internal/to-file.mjs +3 -2
- package/internal/to-file.mjs.map +1 -1
- package/internal/uploads.d.mts +1 -1
- package/internal/uploads.d.mts.map +1 -1
- package/internal/uploads.d.ts +1 -1
- package/internal/uploads.d.ts.map +1 -1
- package/internal/uploads.js +14 -4
- package/internal/uploads.js.map +1 -1
- package/internal/uploads.mjs +12 -3
- package/internal/uploads.mjs.map +1 -1
- package/internal/utils/uuid.d.mts +1 -1
- package/internal/utils/uuid.d.mts.map +1 -1
- package/internal/utils/uuid.d.ts +1 -1
- package/internal/utils/uuid.d.ts.map +1 -1
- package/internal/utils/uuid.js +1 -2
- package/internal/utils/uuid.js.map +1 -1
- package/internal/utils/uuid.mjs +1 -2
- package/internal/utils/uuid.mjs.map +1 -1
- package/package.json +1 -5
- package/resources/index.d.mts +2 -1
- package/resources/index.d.mts.map +1 -1
- package/resources/index.d.ts +2 -1
- package/resources/index.d.ts.map +1 -1
- package/resources/index.js +3 -1
- package/resources/index.js.map +1 -1
- package/resources/index.mjs +1 -0
- package/resources/index.mjs.map +1 -1
- package/resources/projects.d.mts +39 -4
- package/resources/projects.d.mts.map +1 -1
- package/resources/projects.d.ts +39 -4
- package/resources/projects.d.ts.map +1 -1
- package/resources/projects.js +23 -4
- package/resources/projects.js.map +1 -1
- package/resources/projects.mjs +23 -4
- package/resources/projects.mjs.map +1 -1
- package/resources/records.d.mts +10 -0
- package/resources/records.d.mts.map +1 -1
- package/resources/records.d.ts +10 -0
- package/resources/records.d.ts.map +1 -1
- package/resources/records.js +10 -0
- package/resources/records.js.map +1 -1
- package/resources/records.mjs +10 -0
- package/resources/records.mjs.map +1 -1
- package/resources/runs.d.mts +16 -0
- package/resources/runs.d.mts.map +1 -1
- package/resources/runs.d.ts +16 -0
- package/resources/runs.d.ts.map +1 -1
- package/resources/runs.js +16 -0
- package/resources/runs.js.map +1 -1
- package/resources/runs.mjs +16 -0
- package/resources/runs.mjs.map +1 -1
- package/resources/scores.d.mts +83 -0
- package/resources/scores.d.mts.map +1 -0
- package/resources/scores.d.ts +83 -0
- package/resources/scores.d.ts.map +1 -0
- package/resources/scores.js +35 -0
- package/resources/scores.js.map +1 -0
- package/resources/scores.mjs +31 -0
- package/resources/scores.mjs.map +1 -0
- package/resources/system-configs.d.mts +33 -0
- package/resources/system-configs.d.mts.map +1 -1
- package/resources/system-configs.d.ts +33 -0
- package/resources/system-configs.d.ts.map +1 -1
- package/resources/system-configs.js +33 -0
- package/resources/system-configs.js.map +1 -1
- package/resources/system-configs.mjs +33 -0
- package/resources/system-configs.mjs.map +1 -1
- package/resources/systems.d.mts +73 -0
- package/resources/systems.d.mts.map +1 -1
- package/resources/systems.d.ts +73 -0
- package/resources/systems.d.ts.map +1 -1
- package/resources/systems.js +73 -0
- package/resources/systems.js.map +1 -1
- package/resources/systems.mjs +73 -0
- package/resources/systems.mjs.map +1 -1
- package/resources/testcases.d.mts +62 -0
- package/resources/testcases.d.mts.map +1 -1
- package/resources/testcases.d.ts +62 -0
- package/resources/testcases.d.ts.map +1 -1
- package/resources/testcases.js +62 -0
- package/resources/testcases.js.map +1 -1
- package/resources/testcases.mjs +62 -0
- package/resources/testcases.mjs.map +1 -1
- package/resources/testsets.d.mts +48 -0
- package/resources/testsets.d.mts.map +1 -1
- package/resources/testsets.d.ts +48 -0
- package/resources/testsets.d.ts.map +1 -1
- package/resources/testsets.js +48 -0
- package/resources/testsets.js.map +1 -1
- package/resources/testsets.mjs +48 -0
- package/resources/testsets.mjs.map +1 -1
- package/src/client.ts +38 -13
- package/src/internal/headers.ts +1 -1
- package/src/internal/to-file.ts +4 -2
- package/src/internal/uploads.ts +16 -4
- package/src/internal/utils/uuid.ts +1 -3
- package/src/resources/index.ts +4 -2
- package/src/resources/projects.ts +48 -9
- package/src/resources/records.ts +10 -0
- package/src/resources/runs.ts +16 -0
- package/src/resources/scores.ts +98 -0
- package/src/resources/system-configs.ts +33 -0
- package/src/resources/systems.ts +73 -0
- package/src/resources/testcases.ts +62 -0
- package/src/resources/testsets.ts +48 -0
- package/src/version.ts +1 -1
- package/version.d.mts +1 -1
- package/version.d.ts +1 -1
- package/version.js +1 -1
- package/version.mjs +1 -1
- package/internal/shims/crypto.d.mts +0 -13
- package/internal/shims/crypto.d.mts.map +0 -1
- package/internal/shims/crypto.d.ts +0 -13
- package/internal/shims/crypto.d.ts.map +0 -1
- package/internal/shims/crypto.js +0 -13
- package/internal/shims/crypto.js.map +0 -1
- package/internal/shims/crypto.mjs +0 -9
- package/internal/shims/crypto.mjs.map +0 -1
- package/internal/shims/file.d.mts +0 -22
- package/internal/shims/file.d.mts.map +0 -1
- package/internal/shims/file.d.ts +0 -22
- package/internal/shims/file.d.ts.map +0 -1
- package/internal/shims/file.js +0 -16
- package/internal/shims/file.js.map +0 -1
- package/internal/shims/file.mjs +0 -12
- package/internal/shims/file.mjs.map +0 -1
- package/internal/shims/getBuiltinModule.d.mts +0 -11
- package/internal/shims/getBuiltinModule.d.mts.map +0 -1
- package/internal/shims/getBuiltinModule.d.ts +0 -11
- package/internal/shims/getBuiltinModule.d.ts.map +0 -1
- package/internal/shims/getBuiltinModule.js +0 -71
- package/internal/shims/getBuiltinModule.js.map +0 -1
- package/internal/shims/getBuiltinModule.mjs +0 -67
- package/internal/shims/getBuiltinModule.mjs.map +0 -1
- package/internal/shims/nullGetBuiltinModule.d.mts +0 -2
- package/internal/shims/nullGetBuiltinModule.d.mts.map +0 -1
- package/internal/shims/nullGetBuiltinModule.d.ts +0 -2
- package/internal/shims/nullGetBuiltinModule.d.ts.map +0 -1
- package/internal/shims/nullGetBuiltinModule.js +0 -5
- package/internal/shims/nullGetBuiltinModule.js.map +0 -1
- package/internal/shims/nullGetBuiltinModule.mjs +0 -2
- package/internal/shims/nullGetBuiltinModule.mjs.map +0 -1
- package/src/internal/shims/crypto.ts +0 -18
- package/src/internal/shims/file.ts +0 -32
- package/src/internal/shims/getBuiltinModule.ts +0 -66
- package/src/internal/shims/nullGetBuiltinModule.ts +0 -1
|
@@ -1,48 +1,87 @@
|
|
|
1
1
|
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
2
2
|
|
|
3
3
|
import { APIResource } from '../core/resource';
|
|
4
|
+
import { APIPromise } from '../core/api-promise';
|
|
4
5
|
import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../core/pagination';
|
|
5
6
|
import { RequestOptions } from '../internal/request-options';
|
|
6
7
|
|
|
7
8
|
export class Projects extends APIResource {
|
|
9
|
+
/**
|
|
10
|
+
* Create a new Project.
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* ```ts
|
|
14
|
+
* const project = await client.projects.create({
|
|
15
|
+
* description: 'This is a test project',
|
|
16
|
+
* name: 'My Project',
|
|
17
|
+
* });
|
|
18
|
+
* ```
|
|
19
|
+
*/
|
|
20
|
+
create(body: ProjectCreateParams, options?: RequestOptions): APIPromise<Project> {
|
|
21
|
+
return this._client.post('/projects', { body, ...options });
|
|
22
|
+
}
|
|
23
|
+
|
|
8
24
|
/**
|
|
9
25
|
* Retrieve a paginated list of all Projects. Projects are ordered by creation
|
|
10
26
|
* date, with oldest Projects first.
|
|
27
|
+
*
|
|
28
|
+
* @example
|
|
29
|
+
* ```ts
|
|
30
|
+
* // Automatically fetches more pages as needed.
|
|
31
|
+
* for await (const project of client.projects.list()) {
|
|
32
|
+
* // ...
|
|
33
|
+
* }
|
|
34
|
+
* ```
|
|
11
35
|
*/
|
|
12
36
|
list(
|
|
13
37
|
query: ProjectListParams | null | undefined = {},
|
|
14
38
|
options?: RequestOptions,
|
|
15
|
-
): PagePromise<
|
|
16
|
-
return this._client.getAPIList('/projects', PaginatedResponse<
|
|
17
|
-
query,
|
|
18
|
-
...options,
|
|
19
|
-
});
|
|
39
|
+
): PagePromise<ProjectsPaginatedResponse, Project> {
|
|
40
|
+
return this._client.getAPIList('/projects', PaginatedResponse<Project>, { query, ...options });
|
|
20
41
|
}
|
|
21
42
|
}
|
|
22
43
|
|
|
23
|
-
export type
|
|
44
|
+
export type ProjectsPaginatedResponse = PaginatedResponse<Project>;
|
|
24
45
|
|
|
25
46
|
/**
|
|
26
47
|
* A Project in the Scorecard system.
|
|
27
48
|
*/
|
|
28
|
-
export interface
|
|
49
|
+
export interface Project {
|
|
29
50
|
/**
|
|
30
51
|
* The ID of the Project.
|
|
31
52
|
*/
|
|
32
53
|
id: string;
|
|
33
54
|
|
|
55
|
+
/**
|
|
56
|
+
* The description of the Project.
|
|
57
|
+
*/
|
|
58
|
+
description: string | null;
|
|
59
|
+
|
|
34
60
|
/**
|
|
35
61
|
* The name of the Project.
|
|
36
62
|
*/
|
|
37
63
|
name: string | null;
|
|
38
64
|
}
|
|
39
65
|
|
|
66
|
+
export interface ProjectCreateParams {
|
|
67
|
+
/**
|
|
68
|
+
* The description of the Project.
|
|
69
|
+
*/
|
|
70
|
+
description: string;
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* The name of the Project.
|
|
74
|
+
*/
|
|
75
|
+
name: string;
|
|
76
|
+
}
|
|
77
|
+
|
|
40
78
|
export interface ProjectListParams extends PaginatedResponseParams {}
|
|
41
79
|
|
|
42
80
|
export declare namespace Projects {
|
|
43
81
|
export {
|
|
44
|
-
type
|
|
45
|
-
type
|
|
82
|
+
type Project as Project,
|
|
83
|
+
type ProjectsPaginatedResponse as ProjectsPaginatedResponse,
|
|
84
|
+
type ProjectCreateParams as ProjectCreateParams,
|
|
46
85
|
type ProjectListParams as ProjectListParams,
|
|
47
86
|
};
|
|
48
87
|
}
|
package/src/resources/records.ts
CHANGED
|
@@ -9,6 +9,16 @@ import { path } from '../internal/utils/path';
|
|
|
9
9
|
export class Records extends APIResource {
|
|
10
10
|
/**
|
|
11
11
|
* Create a new Record in a Run.
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* ```ts
|
|
15
|
+
* const record = await client.records.create('135', {
|
|
16
|
+
* inputs: { question: 'What is the capital of France?' },
|
|
17
|
+
* labels: { idealAnswer: 'Paris is the capital of France' },
|
|
18
|
+
* outputs: { response: 'The capital of France is Paris.' },
|
|
19
|
+
* testcaseId: '248',
|
|
20
|
+
* });
|
|
21
|
+
* ```
|
|
12
22
|
*/
|
|
13
23
|
create(runID: string, body: RecordCreateParams, options?: RequestOptions): APIPromise<Record> {
|
|
14
24
|
return this._client.post(path`/runs/${runID}/records`, { body, ...options });
|
package/src/resources/runs.ts
CHANGED
|
@@ -8,6 +8,15 @@ import { path } from '../internal/utils/path';
|
|
|
8
8
|
export class Runs extends APIResource {
|
|
9
9
|
/**
|
|
10
10
|
* Create a new Run.
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* ```ts
|
|
14
|
+
* const run = await client.runs.create('314', {
|
|
15
|
+
* metricIds: ['789', '101'],
|
|
16
|
+
* testsetId: '246',
|
|
17
|
+
* systemConfigId: '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
|
|
18
|
+
* });
|
|
19
|
+
* ```
|
|
11
20
|
*/
|
|
12
21
|
create(projectID: string, body: RunCreateParams, options?: RequestOptions): APIPromise<Run> {
|
|
13
22
|
return this._client.post(path`/projects/${projectID}/runs`, { body, ...options });
|
|
@@ -15,6 +24,13 @@ export class Runs extends APIResource {
|
|
|
15
24
|
|
|
16
25
|
/**
|
|
17
26
|
* Update the status of a Run.
|
|
27
|
+
*
|
|
28
|
+
* @example
|
|
29
|
+
* ```ts
|
|
30
|
+
* const run = await client.runs.update('135', {
|
|
31
|
+
* status: 'awaiting_scoring',
|
|
32
|
+
* });
|
|
33
|
+
* ```
|
|
18
34
|
*/
|
|
19
35
|
update(runID: string, body: RunUpdateParams, options?: RequestOptions): APIPromise<RunUpdateResponse> {
|
|
20
36
|
return this._client.patch(path`/runs/${runID}`, { body, ...options });
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
2
|
+
|
|
3
|
+
import { APIResource } from '../core/resource';
|
|
4
|
+
import { APIPromise } from '../core/api-promise';
|
|
5
|
+
import { RequestOptions } from '../internal/request-options';
|
|
6
|
+
import { path } from '../internal/utils/path';
|
|
7
|
+
|
|
8
|
+
export class Scores extends APIResource {
|
|
9
|
+
/**
|
|
10
|
+
* Create or update a Score for a given Record and MetricConfig. If a Score with
|
|
11
|
+
* the specified Record ID and MetricConfig ID already exists, it will be updated.
|
|
12
|
+
* Otherwise, a new Score will be created. The score provided should conform to the
|
|
13
|
+
* schema defined by the MetricConfig; otherwise, validation errors will be
|
|
14
|
+
* reported.
|
|
15
|
+
*
|
|
16
|
+
* @example
|
|
17
|
+
* ```ts
|
|
18
|
+
* const score = await client.scores.upsert(
|
|
19
|
+
* 'a1b2c3d4-e5f6-7890-1234-567890abcdef',
|
|
20
|
+
* {
|
|
21
|
+
* recordId: '777',
|
|
22
|
+
* score: {
|
|
23
|
+
* value: true,
|
|
24
|
+
* reasoning: 'The response is correct',
|
|
25
|
+
* },
|
|
26
|
+
* },
|
|
27
|
+
* );
|
|
28
|
+
* ```
|
|
29
|
+
*/
|
|
30
|
+
upsert(metricConfigID: string, params: ScoreUpsertParams, options?: RequestOptions): APIPromise<Score> {
|
|
31
|
+
const { recordId, ...body } = params;
|
|
32
|
+
return this._client.put(path`/records/${recordId}/scores/${metricConfigID}`, { body, ...options });
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* A Score represents the evaluation of a Record against a specific MetricConfig.
|
|
38
|
+
* The actual `score` is stored as flexible JSON. While any JSON is accepted, it is
|
|
39
|
+
* expected to conform to the output schema defined by the MetricConfig. Any
|
|
40
|
+
* discrepancies will be noted in the `validationErrors` field, but the Score will
|
|
41
|
+
* still be stored.
|
|
42
|
+
*/
|
|
43
|
+
export interface Score {
|
|
44
|
+
/**
|
|
45
|
+
* The ID of the MetricConfig this Score is for.
|
|
46
|
+
*/
|
|
47
|
+
metricConfigId: string;
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* The ID of the Record this Score is for.
|
|
51
|
+
*/
|
|
52
|
+
recordId: string;
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The score of the Record, as arbitrary JSON. This data should ideally conform to
|
|
56
|
+
* the output schema defined by the associated MetricConfig. If it doesn't,
|
|
57
|
+
* validation errors will be captured in the `validationErrors` field.
|
|
58
|
+
*/
|
|
59
|
+
score: Record<string, unknown>;
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Validation errors found in the Score data. If present, the Score doesn't fully
|
|
63
|
+
* conform to its MetricConfig's schema.
|
|
64
|
+
*/
|
|
65
|
+
validationErrors?: Array<Score.ValidationError>;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export namespace Score {
|
|
69
|
+
export interface ValidationError {
|
|
70
|
+
/**
|
|
71
|
+
* Human-readable error description.
|
|
72
|
+
*/
|
|
73
|
+
message: string;
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* JSON Pointer to the field with the validation error.
|
|
77
|
+
*/
|
|
78
|
+
path: string;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export interface ScoreUpsertParams {
|
|
83
|
+
/**
|
|
84
|
+
* Path param: The ID of the Record.
|
|
85
|
+
*/
|
|
86
|
+
recordId: string;
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Body param: The score of the Record, as arbitrary JSON. This data should ideally
|
|
90
|
+
* conform to the output schema defined by the associated MetricConfig. If it
|
|
91
|
+
* doesn't, validation errors will be captured in the `validationErrors` field.
|
|
92
|
+
*/
|
|
93
|
+
score: Record<string, unknown>;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export declare namespace Scores {
|
|
97
|
+
export { type Score as Score, type ScoreUpsertParams as ScoreUpsertParams };
|
|
98
|
+
}
|
|
@@ -23,6 +23,21 @@ export class SystemConfigs extends APIResource {
|
|
|
23
23
|
* - Validation errors indicate fields that don't match the schema but don't
|
|
24
24
|
* prevent creation
|
|
25
25
|
* - Having validation errors may affect how some evaluation metrics are calculated
|
|
26
|
+
*
|
|
27
|
+
* @example
|
|
28
|
+
* ```ts
|
|
29
|
+
* const systemConfig = await client.systemConfigs.create(
|
|
30
|
+
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
31
|
+
* {
|
|
32
|
+
* config: {
|
|
33
|
+
* temperature: 0.1,
|
|
34
|
+
* maxTokens: 1024,
|
|
35
|
+
* model: 'gpt-4-turbo',
|
|
36
|
+
* },
|
|
37
|
+
* name: 'Production (Low Temperature)',
|
|
38
|
+
* },
|
|
39
|
+
* );
|
|
40
|
+
* ```
|
|
26
41
|
*/
|
|
27
42
|
create(
|
|
28
43
|
systemID: string,
|
|
@@ -37,6 +52,16 @@ export class SystemConfigs extends APIResource {
|
|
|
37
52
|
*
|
|
38
53
|
* System configurations provide concrete parameter values for a System Under Test,
|
|
39
54
|
* defining exactly how the system should be configured during an evaluation run.
|
|
55
|
+
*
|
|
56
|
+
* @example
|
|
57
|
+
* ```ts
|
|
58
|
+
* // Automatically fetches more pages as needed.
|
|
59
|
+
* for await (const systemConfig of client.systemConfigs.list(
|
|
60
|
+
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
61
|
+
* )) {
|
|
62
|
+
* // ...
|
|
63
|
+
* }
|
|
64
|
+
* ```
|
|
40
65
|
*/
|
|
41
66
|
list(
|
|
42
67
|
systemID: string,
|
|
@@ -51,6 +76,14 @@ export class SystemConfigs extends APIResource {
|
|
|
51
76
|
|
|
52
77
|
/**
|
|
53
78
|
* Retrieve a specific system configuration by ID.
|
|
79
|
+
*
|
|
80
|
+
* @example
|
|
81
|
+
* ```ts
|
|
82
|
+
* const systemConfig = await client.systemConfigs.get(
|
|
83
|
+
* '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
|
|
84
|
+
* { systemId: '12345678-0a8b-4f66-b6f3-2ddcfa097257' },
|
|
85
|
+
* );
|
|
86
|
+
* ```
|
|
54
87
|
*/
|
|
55
88
|
get(
|
|
56
89
|
systemConfigID: string,
|
package/src/resources/systems.ts
CHANGED
|
@@ -23,6 +23,45 @@ export class Systems extends APIResource {
|
|
|
23
23
|
*
|
|
24
24
|
* This separation lets you evaluate any system as a black box, focusing on its
|
|
25
25
|
* interface rather than implementation details.
|
|
26
|
+
*
|
|
27
|
+
* @example
|
|
28
|
+
* ```ts
|
|
29
|
+
* const system = await client.systems.create('314', {
|
|
30
|
+
* configSchema: {
|
|
31
|
+
* type: 'object',
|
|
32
|
+
* properties: {
|
|
33
|
+
* temperature: { type: 'number' },
|
|
34
|
+
* maxTokens: { type: 'integer' },
|
|
35
|
+
* model: { type: 'string', enum: ['gpt-4', 'gpt-4-turbo'] },
|
|
36
|
+
* },
|
|
37
|
+
* required: ['model'],
|
|
38
|
+
* },
|
|
39
|
+
* description: 'Production chatbot powered by GPT-4',
|
|
40
|
+
* inputSchema: {
|
|
41
|
+
* type: 'object',
|
|
42
|
+
* properties: {
|
|
43
|
+
* messages: {
|
|
44
|
+
* type: 'array',
|
|
45
|
+
* items: {
|
|
46
|
+
* type: 'object',
|
|
47
|
+
* properties: {
|
|
48
|
+
* role: { type: 'string', enum: ['system', 'user', 'assistant'] },
|
|
49
|
+
* content: { type: 'string' },
|
|
50
|
+
* },
|
|
51
|
+
* required: ['role', 'content'],
|
|
52
|
+
* },
|
|
53
|
+
* },
|
|
54
|
+
* },
|
|
55
|
+
* required: ['messages'],
|
|
56
|
+
* },
|
|
57
|
+
* name: 'GPT-4 Chatbot',
|
|
58
|
+
* outputSchema: {
|
|
59
|
+
* type: 'object',
|
|
60
|
+
* properties: { response: { type: 'string' } },
|
|
61
|
+
* required: ['response'],
|
|
62
|
+
* },
|
|
63
|
+
* });
|
|
64
|
+
* ```
|
|
26
65
|
*/
|
|
27
66
|
create(projectID: string, body: SystemCreateParams, options?: RequestOptions): APIPromise<System> {
|
|
28
67
|
return this._client.post(path`/projects/${projectID}/systems`, { body, ...options });
|
|
@@ -41,6 +80,18 @@ export class Systems extends APIResource {
|
|
|
41
80
|
* - Schema updates won't invalidate existing evaluations or configurations
|
|
42
81
|
* - For significant redesigns, creating a new system definition provides a cleaner
|
|
43
82
|
* separation
|
|
83
|
+
*
|
|
84
|
+
* @example
|
|
85
|
+
* ```ts
|
|
86
|
+
* const system = await client.systems.update(
|
|
87
|
+
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
88
|
+
* {
|
|
89
|
+
* description:
|
|
90
|
+
* 'Updated production chatbot powered by GPT-4 Turbo',
|
|
91
|
+
* name: 'GPT-4 Turbo Chatbot',
|
|
92
|
+
* },
|
|
93
|
+
* );
|
|
94
|
+
* ```
|
|
44
95
|
*/
|
|
45
96
|
update(
|
|
46
97
|
systemID: string,
|
|
@@ -52,6 +103,14 @@ export class Systems extends APIResource {
|
|
|
52
103
|
|
|
53
104
|
/**
|
|
54
105
|
* Retrieve a paginated list of all systems. Systems are ordered by creation date.
|
|
106
|
+
*
|
|
107
|
+
* @example
|
|
108
|
+
* ```ts
|
|
109
|
+
* // Automatically fetches more pages as needed.
|
|
110
|
+
* for await (const system of client.systems.list('314')) {
|
|
111
|
+
* // ...
|
|
112
|
+
* }
|
|
113
|
+
* ```
|
|
55
114
|
*/
|
|
56
115
|
list(
|
|
57
116
|
projectID: string,
|
|
@@ -67,6 +126,13 @@ export class Systems extends APIResource {
|
|
|
67
126
|
/**
|
|
68
127
|
* Delete a system definition by ID. This will not delete associated system
|
|
69
128
|
* configurations.
|
|
129
|
+
*
|
|
130
|
+
* @example
|
|
131
|
+
* ```ts
|
|
132
|
+
* const system = await client.systems.delete(
|
|
133
|
+
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
134
|
+
* );
|
|
135
|
+
* ```
|
|
70
136
|
*/
|
|
71
137
|
delete(systemID: string, options?: RequestOptions): APIPromise<SystemDeleteResponse> {
|
|
72
138
|
return this._client.delete(path`/systems/${systemID}`, options);
|
|
@@ -74,6 +140,13 @@ export class Systems extends APIResource {
|
|
|
74
140
|
|
|
75
141
|
/**
|
|
76
142
|
* Retrieve a specific system by ID.
|
|
143
|
+
*
|
|
144
|
+
* @example
|
|
145
|
+
* ```ts
|
|
146
|
+
* const system = await client.systems.get(
|
|
147
|
+
* '12345678-0a8b-4f66-b6f3-2ddcfa097257',
|
|
148
|
+
* );
|
|
149
|
+
* ```
|
|
77
150
|
*/
|
|
78
151
|
get(systemID: string, options?: RequestOptions): APIPromise<System> {
|
|
79
152
|
return this._client.get(path`/systems/${systemID}`, options);
|
|
@@ -9,6 +9,37 @@ import { path } from '../internal/utils/path';
|
|
|
9
9
|
export class Testcases extends APIResource {
|
|
10
10
|
/**
|
|
11
11
|
* Create multiple Testcases in the specified Testset.
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* ```ts
|
|
15
|
+
* const testcase = await client.testcases.create('246', {
|
|
16
|
+
* items: [
|
|
17
|
+
* {
|
|
18
|
+
* jsonData: {
|
|
19
|
+
* question: 'What is the capital of France?',
|
|
20
|
+
* idealAnswer: 'Paris',
|
|
21
|
+
* provenance: 'hand_curated',
|
|
22
|
+
* },
|
|
23
|
+
* },
|
|
24
|
+
* {
|
|
25
|
+
* jsonData: {
|
|
26
|
+
* question:
|
|
27
|
+
* 'What is the largest planet in our solar system?',
|
|
28
|
+
* idealAnswer: 'Jupiter',
|
|
29
|
+
* provenance: 'synthetic',
|
|
30
|
+
* },
|
|
31
|
+
* },
|
|
32
|
+
* {
|
|
33
|
+
* jsonData: {
|
|
34
|
+
* question:
|
|
35
|
+
* 'How many planets are in our solar system?',
|
|
36
|
+
* idealAnswer: 8,
|
|
37
|
+
* provenance: 'user_feedback',
|
|
38
|
+
* },
|
|
39
|
+
* },
|
|
40
|
+
* ],
|
|
41
|
+
* });
|
|
42
|
+
* ```
|
|
12
43
|
*/
|
|
13
44
|
create(
|
|
14
45
|
testsetID: string,
|
|
@@ -20,6 +51,17 @@ export class Testcases extends APIResource {
|
|
|
20
51
|
|
|
21
52
|
/**
|
|
22
53
|
* Replace the data of an existing Testcase while keeping its ID.
|
|
54
|
+
*
|
|
55
|
+
* @example
|
|
56
|
+
* ```ts
|
|
57
|
+
* const testcase = await client.testcases.update('248', {
|
|
58
|
+
* jsonData: {
|
|
59
|
+
* question: 'What is the capital of France?',
|
|
60
|
+
* idealAnswer: 'Paris is the capital of France',
|
|
61
|
+
* provenance: 'hand_curated',
|
|
62
|
+
* },
|
|
63
|
+
* });
|
|
64
|
+
* ```
|
|
23
65
|
*/
|
|
24
66
|
update(testcaseID: string, body: TestcaseUpdateParams, options?: RequestOptions): APIPromise<Testcase> {
|
|
25
67
|
return this._client.put(path`/testcases/${testcaseID}`, { body, ...options });
|
|
@@ -27,6 +69,14 @@ export class Testcases extends APIResource {
|
|
|
27
69
|
|
|
28
70
|
/**
|
|
29
71
|
* Retrieve a paginated list of Testcases belonging to a Testset.
|
|
72
|
+
*
|
|
73
|
+
* @example
|
|
74
|
+
* ```ts
|
|
75
|
+
* // Automatically fetches more pages as needed.
|
|
76
|
+
* for await (const testcase of client.testcases.list('246')) {
|
|
77
|
+
* // ...
|
|
78
|
+
* }
|
|
79
|
+
* ```
|
|
30
80
|
*/
|
|
31
81
|
list(
|
|
32
82
|
testsetID: string,
|
|
@@ -41,6 +91,13 @@ export class Testcases extends APIResource {
|
|
|
41
91
|
|
|
42
92
|
/**
|
|
43
93
|
* Delete multiple Testcases by their IDs.
|
|
94
|
+
*
|
|
95
|
+
* @example
|
|
96
|
+
* ```ts
|
|
97
|
+
* const testcase = await client.testcases.delete({
|
|
98
|
+
* ids: ['123', '124', '125'],
|
|
99
|
+
* });
|
|
100
|
+
* ```
|
|
44
101
|
*/
|
|
45
102
|
delete(body: TestcaseDeleteParams, options?: RequestOptions): APIPromise<TestcaseDeleteResponse> {
|
|
46
103
|
return this._client.post('/testcases/bulk-delete', { body, ...options });
|
|
@@ -48,6 +105,11 @@ export class Testcases extends APIResource {
|
|
|
48
105
|
|
|
49
106
|
/**
|
|
50
107
|
* Retrieve a specific Testcase by ID.
|
|
108
|
+
*
|
|
109
|
+
* @example
|
|
110
|
+
* ```ts
|
|
111
|
+
* const testcase = await client.testcases.get('248');
|
|
112
|
+
* ```
|
|
51
113
|
*/
|
|
52
114
|
get(testcaseID: string, options?: RequestOptions): APIPromise<Testcase> {
|
|
53
115
|
return this._client.get(path`/testcases/${testcaseID}`, options);
|
|
@@ -10,6 +10,28 @@ export class Testsets extends APIResource {
|
|
|
10
10
|
/**
|
|
11
11
|
* Create a new Testset for a Project. The Testset will be created in the Project
|
|
12
12
|
* specified in the path.
|
|
13
|
+
*
|
|
14
|
+
* @example
|
|
15
|
+
* ```ts
|
|
16
|
+
* const testset = await client.testsets.create('314', {
|
|
17
|
+
* description: 'Testset for long context Q&A chatbot.',
|
|
18
|
+
* fieldMapping: {
|
|
19
|
+
* inputs: ['question'],
|
|
20
|
+
* labels: ['idealAnswer'],
|
|
21
|
+
* metadata: [],
|
|
22
|
+
* },
|
|
23
|
+
* jsonSchema: {
|
|
24
|
+
* type: 'object',
|
|
25
|
+
* properties: {
|
|
26
|
+
* question: { type: 'string' },
|
|
27
|
+
* idealAnswer: { type: 'string' },
|
|
28
|
+
* provenance: { type: 'string' },
|
|
29
|
+
* geo: { type: 'string' },
|
|
30
|
+
* },
|
|
31
|
+
* },
|
|
32
|
+
* name: 'Long Context Q&A',
|
|
33
|
+
* });
|
|
34
|
+
* ```
|
|
13
35
|
*/
|
|
14
36
|
create(projectID: string, body: TestsetCreateParams, options?: RequestOptions): APIPromise<Testset> {
|
|
15
37
|
return this._client.post(path`/projects/${projectID}/testsets`, { body, ...options });
|
|
@@ -28,6 +50,14 @@ export class Testsets extends APIResource {
|
|
|
28
50
|
* updated schema
|
|
29
51
|
* - For complete control, provide both schema and fieldMapping when updating the
|
|
30
52
|
* schema
|
|
53
|
+
*
|
|
54
|
+
* @example
|
|
55
|
+
* ```ts
|
|
56
|
+
* const testset = await client.testsets.update('246', {
|
|
57
|
+
* description: 'Updated description for the Q&A Testset.',
|
|
58
|
+
* name: 'Updated Q&A Testset',
|
|
59
|
+
* });
|
|
60
|
+
* ```
|
|
31
61
|
*/
|
|
32
62
|
update(
|
|
33
63
|
testsetID: string,
|
|
@@ -39,6 +69,14 @@ export class Testsets extends APIResource {
|
|
|
39
69
|
|
|
40
70
|
/**
|
|
41
71
|
* Retrieve a paginated list of Testsets belonging to a Project.
|
|
72
|
+
*
|
|
73
|
+
* @example
|
|
74
|
+
* ```ts
|
|
75
|
+
* // Automatically fetches more pages as needed.
|
|
76
|
+
* for await (const testset of client.testsets.list('314')) {
|
|
77
|
+
* // ...
|
|
78
|
+
* }
|
|
79
|
+
* ```
|
|
42
80
|
*/
|
|
43
81
|
list(
|
|
44
82
|
projectID: string,
|
|
@@ -53,6 +91,11 @@ export class Testsets extends APIResource {
|
|
|
53
91
|
|
|
54
92
|
/**
|
|
55
93
|
* Delete Testset
|
|
94
|
+
*
|
|
95
|
+
* @example
|
|
96
|
+
* ```ts
|
|
97
|
+
* const testset = await client.testsets.delete('246');
|
|
98
|
+
* ```
|
|
56
99
|
*/
|
|
57
100
|
delete(testsetID: string, options?: RequestOptions): APIPromise<TestsetDeleteResponse> {
|
|
58
101
|
return this._client.delete(path`/testsets/${testsetID}`, options);
|
|
@@ -60,6 +103,11 @@ export class Testsets extends APIResource {
|
|
|
60
103
|
|
|
61
104
|
/**
|
|
62
105
|
* Get Testset by ID
|
|
106
|
+
*
|
|
107
|
+
* @example
|
|
108
|
+
* ```ts
|
|
109
|
+
* const testset = await client.testsets.get('246');
|
|
110
|
+
* ```
|
|
63
111
|
*/
|
|
64
112
|
get(testsetID: string, options?: RequestOptions): APIPromise<Testset> {
|
|
65
113
|
return this._client.get(path`/testsets/${testsetID}`, options);
|
package/src/version.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const VERSION = '1.0.0-alpha.
|
|
1
|
+
export const VERSION = '1.0.0-alpha.7'; // x-release-please-version
|
package/version.d.mts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export declare const VERSION = "1.0.0-alpha.
|
|
1
|
+
export declare const VERSION = "1.0.0-alpha.7";
|
|
2
2
|
//# sourceMappingURL=version.d.mts.map
|
package/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export declare const VERSION = "1.0.0-alpha.
|
|
1
|
+
export declare const VERSION = "1.0.0-alpha.7";
|
|
2
2
|
//# sourceMappingURL=version.d.ts.map
|
package/version.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.VERSION = void 0;
|
|
4
|
-
exports.VERSION = '1.0.0-alpha.
|
|
4
|
+
exports.VERSION = '1.0.0-alpha.7'; // x-release-please-version
|
|
5
5
|
//# sourceMappingURL=version.js.map
|
package/version.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export const VERSION = '1.0.0-alpha.
|
|
1
|
+
export const VERSION = '1.0.0-alpha.7'; // x-release-please-version
|
|
2
2
|
//# sourceMappingURL=version.mjs.map
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
type Crypto = {
|
|
2
|
-
/** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/getRandomValues) */
|
|
3
|
-
getRandomValues<T extends ArrayBufferView | null>(array: T): T;
|
|
4
|
-
/**
|
|
5
|
-
* Available only in secure contexts.
|
|
6
|
-
*
|
|
7
|
-
* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/randomUUID)
|
|
8
|
-
*/
|
|
9
|
-
randomUUID?: () => string;
|
|
10
|
-
};
|
|
11
|
-
export declare let getCrypto: () => Crypto | undefined;
|
|
12
|
-
export {};
|
|
13
|
-
//# sourceMappingURL=crypto.d.mts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"crypto.d.mts","sourceRoot":"","sources":["../../src/internal/shims/crypto.ts"],"names":[],"mappings":"AAEA,KAAK,MAAM,GAAG;IACZ,yFAAyF;IACzF,eAAe,CAAC,CAAC,SAAS,eAAe,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC;IAC/D;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,MAAM,CAAC;CAC3B,CAAC;AACF,eAAO,IAAI,SAAS,EAAE,MAAM,MAAM,GAAG,SAKpC,CAAC"}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
type Crypto = {
|
|
2
|
-
/** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/getRandomValues) */
|
|
3
|
-
getRandomValues<T extends ArrayBufferView | null>(array: T): T;
|
|
4
|
-
/**
|
|
5
|
-
* Available only in secure contexts.
|
|
6
|
-
*
|
|
7
|
-
* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/randomUUID)
|
|
8
|
-
*/
|
|
9
|
-
randomUUID?: () => string;
|
|
10
|
-
};
|
|
11
|
-
export declare let getCrypto: () => Crypto | undefined;
|
|
12
|
-
export {};
|
|
13
|
-
//# sourceMappingURL=crypto.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"crypto.d.ts","sourceRoot":"","sources":["../../src/internal/shims/crypto.ts"],"names":[],"mappings":"AAEA,KAAK,MAAM,GAAG;IACZ,yFAAyF;IACzF,eAAe,CAAC,CAAC,SAAS,eAAe,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC;IAC/D;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,MAAM,CAAC;CAC3B,CAAC;AACF,eAAO,IAAI,SAAS,EAAE,MAAM,MAAM,GAAG,SAKpC,CAAC"}
|
package/internal/shims/crypto.js
DELETED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.getCrypto = void 0;
|
|
4
|
-
const getBuiltinModule_1 = require("./getBuiltinModule.js");
|
|
5
|
-
let getCrypto = function lazyGetCrypto() {
|
|
6
|
-
if (exports.getCrypto !== lazyGetCrypto)
|
|
7
|
-
return (0, exports.getCrypto)();
|
|
8
|
-
const crypto = globalThis.crypto || (0, getBuiltinModule_1.getBuiltinModule)?.('node:crypto')?.webcrypto;
|
|
9
|
-
exports.getCrypto = () => crypto;
|
|
10
|
-
return crypto;
|
|
11
|
-
};
|
|
12
|
-
exports.getCrypto = getCrypto;
|
|
13
|
-
//# sourceMappingURL=crypto.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"crypto.js","sourceRoot":"","sources":["../../src/internal/shims/crypto.ts"],"names":[],"mappings":";;;AAAA,4DAAsD;AAY/C,IAAI,SAAS,GAA6B,SAAS,aAAa;IACrE,IAAI,iBAAS,KAAK,aAAa;QAAE,OAAO,IAAA,iBAAS,GAAE,CAAC;IACpD,MAAM,MAAM,GAAY,UAAkB,CAAC,MAAM,IAAK,IAAA,mCAAgB,CAAA,EAAE,CAAC,aAAa,CAAS,EAAE,SAAS,CAAC;IAC3G,iBAAS,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AALS,QAAA,SAAS,aAKlB"}
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import { getBuiltinModule } from "./getBuiltinModule.mjs";
|
|
2
|
-
export let getCrypto = function lazyGetCrypto() {
|
|
3
|
-
if (getCrypto !== lazyGetCrypto)
|
|
4
|
-
return getCrypto();
|
|
5
|
-
const crypto = globalThis.crypto || getBuiltinModule?.('node:crypto')?.webcrypto;
|
|
6
|
-
getCrypto = () => crypto;
|
|
7
|
-
return crypto;
|
|
8
|
-
};
|
|
9
|
-
//# sourceMappingURL=crypto.mjs.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"crypto.mjs","sourceRoot":"","sources":["../../src/internal/shims/crypto.ts"],"names":[],"mappings":"OAAO,EAAE,gBAAgB,EAAE;AAY3B,MAAM,CAAC,IAAI,SAAS,GAA6B,SAAS,aAAa;IACrE,IAAI,SAAS,KAAK,aAAa;QAAE,OAAO,SAAS,EAAE,CAAC;IACpD,MAAM,MAAM,GAAY,UAAkB,CAAC,MAAM,IAAK,gBAAgB,EAAE,CAAC,aAAa,CAAS,EAAE,SAAS,CAAC;IAC3G,SAAS,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC"}
|