scorecard-ai 0.3.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Client.d.ts +2 -0
- package/api/resources/prompt/client/Client.d.ts +22 -40
- package/api/resources/prompt/client/Client.js +50 -205
- package/api/resources/prompt/client/requests/PromptCreateParams.d.ts +2 -2
- package/api/resources/prompt/client/requests/PromptUpdateParams.d.ts +1 -2
- package/api/resources/run/client/Client.d.ts +5 -13
- package/api/resources/run/client/Client.js +18 -25
- package/api/resources/run/client/requests/RunCreateParams.d.ts +1 -9
- package/api/resources/run/client/requests/UpdateStatusParams.d.ts +2 -4
- package/api/resources/runMetric/client/Client.d.ts +2 -0
- package/api/resources/runMetric/client/Client.js +5 -4
- package/api/resources/score/client/Client.d.ts +7 -5
- package/api/resources/score/client/Client.js +15 -13
- package/api/resources/score/client/requests/ScoreCreateParams.d.ts +1 -0
- package/api/resources/testcase/client/Client.d.ts +10 -10
- package/api/resources/testcase/client/Client.js +29 -22
- package/api/resources/testcase/client/requests/TestcaseCreateParams.d.ts +3 -4
- package/api/resources/testcase/types/TestcaseCreateParamsCustomInputsValue.d.ts +1 -1
- package/api/resources/testcase/types/TestcaseCreateParamsCustomLabelsValue.d.ts +1 -1
- package/api/resources/testrecord/client/Client.d.ts +3 -1
- package/api/resources/testrecord/client/Client.js +11 -9
- package/api/resources/testset/client/Client.d.ts +6 -4
- package/api/resources/testset/client/Client.js +29 -24
- package/api/resources/testset/client/requests/TestsetGetTestcasesRequest.d.ts +6 -0
- package/api/resources/traces/client/Client.d.ts +2 -0
- package/api/resources/traces/client/Client.js +5 -4
- package/api/types/Grade.d.ts +2 -0
- package/api/types/PaginatedTestcaseResponse.d.ts +1 -0
- package/api/types/TestCase.d.ts +2 -0
- package/api/types/TestCaseCustomInputsValue.d.ts +1 -1
- package/api/types/TestCaseCustomLabelsValue.d.ts +1 -1
- package/api/types/TestcaseDeletionResponse.d.ts +9 -0
- package/api/types/TestcaseDeletionResponse.js +5 -0
- package/api/types/Testrecord.d.ts +0 -1
- package/api/types/index.d.ts +1 -0
- package/api/types/index.js +1 -0
- package/core/fetcher/Fetcher.d.ts +1 -0
- package/core/fetcher/Fetcher.js +58 -7
- package/dist/Client.d.ts +2 -0
- package/dist/api/resources/prompt/client/Client.d.ts +22 -40
- package/dist/api/resources/prompt/client/Client.js +50 -205
- package/dist/api/resources/prompt/client/requests/PromptCreateParams.d.ts +2 -2
- package/dist/api/resources/prompt/client/requests/PromptUpdateParams.d.ts +1 -2
- package/dist/api/resources/run/client/Client.d.ts +5 -13
- package/dist/api/resources/run/client/Client.js +18 -25
- package/dist/api/resources/run/client/requests/RunCreateParams.d.ts +1 -9
- package/dist/api/resources/run/client/requests/UpdateStatusParams.d.ts +2 -4
- package/dist/api/resources/runMetric/client/Client.d.ts +2 -0
- package/dist/api/resources/runMetric/client/Client.js +5 -4
- package/dist/api/resources/score/client/Client.d.ts +7 -5
- package/dist/api/resources/score/client/Client.js +15 -13
- package/dist/api/resources/score/client/requests/ScoreCreateParams.d.ts +1 -0
- package/dist/api/resources/testcase/client/Client.d.ts +10 -10
- package/dist/api/resources/testcase/client/Client.js +29 -22
- package/dist/api/resources/testcase/client/requests/TestcaseCreateParams.d.ts +3 -4
- package/dist/api/resources/testcase/types/TestcaseCreateParamsCustomInputsValue.d.ts +1 -1
- package/dist/api/resources/testcase/types/TestcaseCreateParamsCustomLabelsValue.d.ts +1 -1
- package/dist/api/resources/testrecord/client/Client.d.ts +3 -1
- package/dist/api/resources/testrecord/client/Client.js +11 -9
- package/dist/api/resources/testset/client/Client.d.ts +6 -4
- package/dist/api/resources/testset/client/Client.js +29 -24
- package/dist/api/resources/testset/client/requests/TestsetGetTestcasesRequest.d.ts +6 -0
- package/dist/api/resources/traces/client/Client.d.ts +2 -0
- package/dist/api/resources/traces/client/Client.js +5 -4
- package/dist/api/types/Grade.d.ts +2 -0
- package/dist/api/types/PaginatedTestcaseResponse.d.ts +1 -0
- package/dist/api/types/TestCase.d.ts +2 -0
- package/dist/api/types/TestCaseCustomInputsValue.d.ts +1 -1
- package/dist/api/types/TestCaseCustomLabelsValue.d.ts +1 -1
- package/dist/api/types/TestcaseDeletionResponse.d.ts +9 -0
- package/dist/api/types/TestcaseDeletionResponse.js +5 -0
- package/dist/api/types/Testrecord.d.ts +0 -1
- package/dist/api/types/index.d.ts +1 -0
- package/dist/api/types/index.js +1 -0
- package/dist/core/fetcher/Fetcher.d.ts +1 -0
- package/dist/core/fetcher/Fetcher.js +58 -7
- package/dist/serialization/resources/prompt/client/index.d.ts +0 -3
- package/dist/serialization/resources/prompt/client/index.js +0 -16
- package/dist/serialization/resources/prompt/client/requests/PromptUpdateParams.d.ts +0 -1
- package/dist/serialization/resources/prompt/client/requests/PromptUpdateParams.js +0 -1
- package/dist/serialization/resources/run/client/requests/UpdateStatusParams.d.ts +1 -1
- package/dist/serialization/resources/run/client/requests/UpdateStatusParams.js +1 -1
- package/dist/serialization/resources/testcase/client/requests/TestcaseCreateParams.d.ts +1 -1
- package/dist/serialization/resources/testcase/client/requests/TestcaseCreateParams.js +1 -1
- package/dist/serialization/resources/testcase/types/TestcaseCreateParamsCustomInputsValue.d.ts +1 -1
- package/dist/serialization/resources/testcase/types/TestcaseCreateParamsCustomInputsValue.js +4 -1
- package/dist/serialization/resources/testcase/types/TestcaseCreateParamsCustomLabelsValue.d.ts +1 -1
- package/dist/serialization/resources/testcase/types/TestcaseCreateParamsCustomLabelsValue.js +4 -1
- package/dist/serialization/types/TestCaseCustomInputsValue.d.ts +1 -1
- package/dist/serialization/types/TestCaseCustomInputsValue.js +4 -1
- package/dist/serialization/types/TestCaseCustomLabelsValue.d.ts +1 -1
- package/dist/serialization/types/TestCaseCustomLabelsValue.js +4 -1
- package/dist/serialization/types/TestcaseDeletionResponse.d.ts +13 -0
- package/dist/serialization/{resources/prompt/client/getRoots.js → types/TestcaseDeletionResponse.js} +6 -4
- package/dist/serialization/types/Testrecord.d.ts +0 -1
- package/dist/serialization/types/Testrecord.js +0 -1
- package/dist/serialization/types/index.d.ts +1 -0
- package/dist/serialization/types/index.js +1 -0
- package/dist/telemetry.d.ts +6 -0
- package/dist/telemetry.js +65 -0
- package/dist/wrapper/ScorecardClient.d.ts +4 -0
- package/dist/wrapper/ScorecardClient.js +4 -6
- package/package.json +56 -2
- package/reference.md +38 -189
- package/serialization/resources/prompt/client/index.d.ts +0 -3
- package/serialization/resources/prompt/client/index.js +0 -16
- package/serialization/resources/prompt/client/requests/PromptUpdateParams.d.ts +0 -1
- package/serialization/resources/prompt/client/requests/PromptUpdateParams.js +0 -1
- package/serialization/resources/run/client/requests/UpdateStatusParams.d.ts +1 -1
- package/serialization/resources/run/client/requests/UpdateStatusParams.js +1 -1
- package/serialization/resources/testcase/client/requests/TestcaseCreateParams.d.ts +1 -1
- package/serialization/resources/testcase/client/requests/TestcaseCreateParams.js +1 -1
- package/serialization/resources/testcase/types/TestcaseCreateParamsCustomInputsValue.d.ts +1 -1
- package/serialization/resources/testcase/types/TestcaseCreateParamsCustomInputsValue.js +4 -1
- package/serialization/resources/testcase/types/TestcaseCreateParamsCustomLabelsValue.d.ts +1 -1
- package/serialization/resources/testcase/types/TestcaseCreateParamsCustomLabelsValue.js +4 -1
- package/serialization/types/TestCaseCustomInputsValue.d.ts +1 -1
- package/serialization/types/TestCaseCustomInputsValue.js +4 -1
- package/serialization/types/TestCaseCustomLabelsValue.d.ts +1 -1
- package/serialization/types/TestCaseCustomLabelsValue.js +4 -1
- package/serialization/types/TestcaseDeletionResponse.d.ts +13 -0
- package/{dist/serialization/resources/prompt/client/getGraph.js → serialization/types/TestcaseDeletionResponse.js} +6 -4
- package/serialization/types/Testrecord.d.ts +0 -1
- package/serialization/types/Testrecord.js +0 -1
- package/serialization/types/index.d.ts +1 -0
- package/serialization/types/index.js +1 -0
- package/telemetry.d.ts +6 -0
- package/telemetry.js +65 -0
- package/wrapper/ScorecardClient.d.ts +4 -0
- package/wrapper/ScorecardClient.js +4 -6
- package/dist/serialization/resources/prompt/client/getGraph.d.ts +0 -11
- package/dist/serialization/resources/prompt/client/getProds.d.ts +0 -11
- package/dist/serialization/resources/prompt/client/getProds.js +0 -32
- package/dist/serialization/resources/prompt/client/getRoots.d.ts +0 -11
- package/serialization/resources/prompt/client/getGraph.d.ts +0 -11
- package/serialization/resources/prompt/client/getGraph.js +0 -32
- package/serialization/resources/prompt/client/getProds.d.ts +0 -11
- package/serialization/resources/prompt/client/getProds.js +0 -32
- package/serialization/resources/prompt/client/getRoots.d.ts +0 -11
- package/serialization/resources/prompt/client/getRoots.js +0 -32
|
@@ -52,7 +52,7 @@ class Testset {
|
|
|
52
52
|
/**
|
|
53
53
|
* Retrieve Testset metadata without Testcase data
|
|
54
54
|
*
|
|
55
|
-
* @param {number} testsetId
|
|
55
|
+
* @param {number} testsetId - The ID of the Testset to retrieve.
|
|
56
56
|
* @param {Testset.RequestOptions} requestOptions - Request-specific configuration.
|
|
57
57
|
*
|
|
58
58
|
* @throws {@link Scorecard.UnauthorizedError}
|
|
@@ -64,15 +64,16 @@ class Testset {
|
|
|
64
64
|
* await scorecard.testset.get(1)
|
|
65
65
|
*/
|
|
66
66
|
get(testsetId, requestOptions) {
|
|
67
|
-
var _a;
|
|
67
|
+
var _a, _b;
|
|
68
68
|
return __awaiter(this, void 0, void 0, function* () {
|
|
69
|
-
const _response = yield core.fetcher({
|
|
70
|
-
url: (0, url_join_1.default)((
|
|
69
|
+
const _response = yield ((_a = this._options.fetcher) !== null && _a !== void 0 ? _a : core.fetcher)({
|
|
70
|
+
url: (0, url_join_1.default)((_b = (yield core.Supplier.get(this._options.environment))) !== null && _b !== void 0 ? _b : environments.ScorecardEnvironment.Default, `v1/testset/${encodeURIComponent(testsetId)}`),
|
|
71
71
|
method: "GET",
|
|
72
|
-
headers: Object.assign({ "X-Fern-Language": "JavaScript", "X-Fern-SDK-Name": "scorecard-ai", "X-Fern-SDK-Version": "0.
|
|
72
|
+
headers: Object.assign({ "X-Fern-Language": "JavaScript", "X-Fern-SDK-Name": "scorecard-ai", "X-Fern-SDK-Version": "0.5.2", "X-Fern-Runtime": core.RUNTIME.type, "X-Fern-Runtime-Version": core.RUNTIME.version }, (yield this._getCustomAuthorizationHeaders())),
|
|
73
73
|
contentType: "application/json",
|
|
74
74
|
timeoutMs: (requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.timeoutInSeconds) != null ? requestOptions.timeoutInSeconds * 1000 : 60000,
|
|
75
75
|
maxRetries: requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.maxRetries,
|
|
76
|
+
abortSignal: requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.abortSignal,
|
|
76
77
|
});
|
|
77
78
|
if (_response.ok) {
|
|
78
79
|
return yield serializers.Testset.parseOrThrow(_response.body, {
|
|
@@ -142,7 +143,7 @@ class Testset {
|
|
|
142
143
|
/**
|
|
143
144
|
* Delete a Testset
|
|
144
145
|
*
|
|
145
|
-
* @param {number} testsetId
|
|
146
|
+
* @param {number} testsetId - The ID of the Testset to delete.
|
|
146
147
|
* @param {Testset.RequestOptions} requestOptions - Request-specific configuration.
|
|
147
148
|
*
|
|
148
149
|
* @throws {@link Scorecard.UnauthorizedError}
|
|
@@ -154,15 +155,16 @@ class Testset {
|
|
|
154
155
|
* await scorecard.testset.delete(1)
|
|
155
156
|
*/
|
|
156
157
|
delete(testsetId, requestOptions) {
|
|
157
|
-
var _a;
|
|
158
|
+
var _a, _b;
|
|
158
159
|
return __awaiter(this, void 0, void 0, function* () {
|
|
159
|
-
const _response = yield core.fetcher({
|
|
160
|
-
url: (0, url_join_1.default)((
|
|
160
|
+
const _response = yield ((_a = this._options.fetcher) !== null && _a !== void 0 ? _a : core.fetcher)({
|
|
161
|
+
url: (0, url_join_1.default)((_b = (yield core.Supplier.get(this._options.environment))) !== null && _b !== void 0 ? _b : environments.ScorecardEnvironment.Default, `v1/testset/${encodeURIComponent(testsetId)}`),
|
|
161
162
|
method: "DELETE",
|
|
162
|
-
headers: Object.assign({ "X-Fern-Language": "JavaScript", "X-Fern-SDK-Name": "scorecard-ai", "X-Fern-SDK-Version": "0.
|
|
163
|
+
headers: Object.assign({ "X-Fern-Language": "JavaScript", "X-Fern-SDK-Name": "scorecard-ai", "X-Fern-SDK-Version": "0.5.2", "X-Fern-Runtime": core.RUNTIME.type, "X-Fern-Runtime-Version": core.RUNTIME.version }, (yield this._getCustomAuthorizationHeaders())),
|
|
163
164
|
contentType: "application/json",
|
|
164
165
|
timeoutMs: (requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.timeoutInSeconds) != null ? requestOptions.timeoutInSeconds * 1000 : 60000,
|
|
165
166
|
maxRetries: requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.maxRetries,
|
|
167
|
+
abortSignal: requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.abortSignal,
|
|
166
168
|
});
|
|
167
169
|
if (_response.ok) {
|
|
168
170
|
return yield serializers.Testset.parseOrThrow(_response.body, {
|
|
@@ -246,16 +248,17 @@ class Testset {
|
|
|
246
248
|
* })
|
|
247
249
|
*/
|
|
248
250
|
create(request, requestOptions) {
|
|
249
|
-
var _a;
|
|
251
|
+
var _a, _b;
|
|
250
252
|
return __awaiter(this, void 0, void 0, function* () {
|
|
251
|
-
const _response = yield core.fetcher({
|
|
252
|
-
url: (0, url_join_1.default)((
|
|
253
|
+
const _response = yield ((_a = this._options.fetcher) !== null && _a !== void 0 ? _a : core.fetcher)({
|
|
254
|
+
url: (0, url_join_1.default)((_b = (yield core.Supplier.get(this._options.environment))) !== null && _b !== void 0 ? _b : environments.ScorecardEnvironment.Default, "v1/testset"),
|
|
253
255
|
method: "POST",
|
|
254
|
-
headers: Object.assign({ "X-Fern-Language": "JavaScript", "X-Fern-SDK-Name": "scorecard-ai", "X-Fern-SDK-Version": "0.
|
|
256
|
+
headers: Object.assign({ "X-Fern-Language": "JavaScript", "X-Fern-SDK-Name": "scorecard-ai", "X-Fern-SDK-Version": "0.5.2", "X-Fern-Runtime": core.RUNTIME.type, "X-Fern-Runtime-Version": core.RUNTIME.version }, (yield this._getCustomAuthorizationHeaders())),
|
|
255
257
|
contentType: "application/json",
|
|
256
258
|
body: yield serializers.TestsetCreateParams.jsonOrThrow(request, { unrecognizedObjectKeys: "strip" }),
|
|
257
259
|
timeoutMs: (requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.timeoutInSeconds) != null ? requestOptions.timeoutInSeconds * 1000 : 60000,
|
|
258
260
|
maxRetries: requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.maxRetries,
|
|
261
|
+
abortSignal: requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.abortSignal,
|
|
259
262
|
});
|
|
260
263
|
if (_response.ok) {
|
|
261
264
|
return yield serializers.Testset.parseOrThrow(_response.body, {
|
|
@@ -325,7 +328,7 @@ class Testset {
|
|
|
325
328
|
/**
|
|
326
329
|
* Read the schema of a Testset
|
|
327
330
|
*
|
|
328
|
-
* @param {number} testsetId
|
|
331
|
+
* @param {number} testsetId - The ID of the Testset to retrieve the schema from.
|
|
329
332
|
* @param {Testset.RequestOptions} requestOptions - Request-specific configuration.
|
|
330
333
|
*
|
|
331
334
|
* @throws {@link Scorecard.UnauthorizedError}
|
|
@@ -337,15 +340,16 @@ class Testset {
|
|
|
337
340
|
* await scorecard.testset.readSchema(1)
|
|
338
341
|
*/
|
|
339
342
|
readSchema(testsetId, requestOptions) {
|
|
340
|
-
var _a;
|
|
343
|
+
var _a, _b;
|
|
341
344
|
return __awaiter(this, void 0, void 0, function* () {
|
|
342
|
-
const _response = yield core.fetcher({
|
|
343
|
-
url: (0, url_join_1.default)((
|
|
345
|
+
const _response = yield ((_a = this._options.fetcher) !== null && _a !== void 0 ? _a : core.fetcher)({
|
|
346
|
+
url: (0, url_join_1.default)((_b = (yield core.Supplier.get(this._options.environment))) !== null && _b !== void 0 ? _b : environments.ScorecardEnvironment.Default, `v1/testset/${encodeURIComponent(testsetId)}/schema`),
|
|
344
347
|
method: "GET",
|
|
345
|
-
headers: Object.assign({ "X-Fern-Language": "JavaScript", "X-Fern-SDK-Name": "scorecard-ai", "X-Fern-SDK-Version": "0.
|
|
348
|
+
headers: Object.assign({ "X-Fern-Language": "JavaScript", "X-Fern-SDK-Name": "scorecard-ai", "X-Fern-SDK-Version": "0.5.2", "X-Fern-Runtime": core.RUNTIME.type, "X-Fern-Runtime-Version": core.RUNTIME.version }, (yield this._getCustomAuthorizationHeaders())),
|
|
346
349
|
contentType: "application/json",
|
|
347
350
|
timeoutMs: (requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.timeoutInSeconds) != null ? requestOptions.timeoutInSeconds * 1000 : 60000,
|
|
348
351
|
maxRetries: requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.maxRetries,
|
|
352
|
+
abortSignal: requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.abortSignal,
|
|
349
353
|
});
|
|
350
354
|
if (_response.ok) {
|
|
351
355
|
return yield serializers.CustomSchema.parseOrThrow(_response.body, {
|
|
@@ -415,7 +419,7 @@ class Testset {
|
|
|
415
419
|
/**
|
|
416
420
|
* Retrieve all Testcases from a Testset
|
|
417
421
|
*
|
|
418
|
-
* @param {number} testsetId
|
|
422
|
+
* @param {number} testsetId - The Testset ID to retrieve testcases from.
|
|
419
423
|
* @param {Scorecard.TestsetGetTestcasesRequest} request
|
|
420
424
|
* @param {Testset.RequestOptions} requestOptions - Request-specific configuration.
|
|
421
425
|
*
|
|
@@ -428,7 +432,7 @@ class Testset {
|
|
|
428
432
|
* await scorecard.testset.getTestcases(1)
|
|
429
433
|
*/
|
|
430
434
|
getTestcases(testsetId, request = {}, requestOptions) {
|
|
431
|
-
var _a;
|
|
435
|
+
var _a, _b;
|
|
432
436
|
return __awaiter(this, void 0, void 0, function* () {
|
|
433
437
|
const { offset, limit } = request;
|
|
434
438
|
const _queryParams = {};
|
|
@@ -438,14 +442,15 @@ class Testset {
|
|
|
438
442
|
if (limit != null) {
|
|
439
443
|
_queryParams["limit"] = limit.toString();
|
|
440
444
|
}
|
|
441
|
-
const _response = yield core.fetcher({
|
|
442
|
-
url: (0, url_join_1.default)((
|
|
445
|
+
const _response = yield ((_a = this._options.fetcher) !== null && _a !== void 0 ? _a : core.fetcher)({
|
|
446
|
+
url: (0, url_join_1.default)((_b = (yield core.Supplier.get(this._options.environment))) !== null && _b !== void 0 ? _b : environments.ScorecardEnvironment.Default, `v1/testset/${encodeURIComponent(testsetId)}/testcase`),
|
|
443
447
|
method: "GET",
|
|
444
|
-
headers: Object.assign({ "X-Fern-Language": "JavaScript", "X-Fern-SDK-Name": "scorecard-ai", "X-Fern-SDK-Version": "0.
|
|
448
|
+
headers: Object.assign({ "X-Fern-Language": "JavaScript", "X-Fern-SDK-Name": "scorecard-ai", "X-Fern-SDK-Version": "0.5.2", "X-Fern-Runtime": core.RUNTIME.type, "X-Fern-Runtime-Version": core.RUNTIME.version }, (yield this._getCustomAuthorizationHeaders())),
|
|
445
449
|
contentType: "application/json",
|
|
446
450
|
queryParameters: _queryParams,
|
|
447
451
|
timeoutMs: (requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.timeoutInSeconds) != null ? requestOptions.timeoutInSeconds * 1000 : 60000,
|
|
448
452
|
maxRetries: requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.maxRetries,
|
|
453
|
+
abortSignal: requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.abortSignal,
|
|
449
454
|
});
|
|
450
455
|
if (_response.ok) {
|
|
451
456
|
return yield serializers.PaginatedTestcaseResponse.parseOrThrow(_response.body, {
|
|
@@ -8,10 +8,12 @@ export declare namespace Traces {
|
|
|
8
8
|
interface Options {
|
|
9
9
|
environment?: core.Supplier<environments.ScorecardEnvironment | string>;
|
|
10
10
|
apiKey: core.Supplier<string>;
|
|
11
|
+
fetcher?: core.FetchFunction;
|
|
11
12
|
}
|
|
12
13
|
interface RequestOptions {
|
|
13
14
|
timeoutInSeconds?: number;
|
|
14
15
|
maxRetries?: number;
|
|
16
|
+
abortSignal?: AbortSignal;
|
|
15
17
|
}
|
|
16
18
|
}
|
|
17
19
|
export declare class Traces {
|
|
@@ -64,15 +64,16 @@ class Traces {
|
|
|
64
64
|
* await scorecard.traces.get("trace_id")
|
|
65
65
|
*/
|
|
66
66
|
get(traceId, requestOptions) {
|
|
67
|
-
var _a;
|
|
67
|
+
var _a, _b;
|
|
68
68
|
return __awaiter(this, void 0, void 0, function* () {
|
|
69
|
-
const _response = yield core.fetcher({
|
|
70
|
-
url: (0, url_join_1.default)((
|
|
69
|
+
const _response = yield ((_a = this._options.fetcher) !== null && _a !== void 0 ? _a : core.fetcher)({
|
|
70
|
+
url: (0, url_join_1.default)((_b = (yield core.Supplier.get(this._options.environment))) !== null && _b !== void 0 ? _b : environments.ScorecardEnvironment.Default, `v1/traces/${encodeURIComponent(traceId)}/spans`),
|
|
71
71
|
method: "GET",
|
|
72
|
-
headers: Object.assign({ "X-Fern-Language": "JavaScript", "X-Fern-SDK-Name": "scorecard-ai", "X-Fern-SDK-Version": "0.
|
|
72
|
+
headers: Object.assign({ "X-Fern-Language": "JavaScript", "X-Fern-SDK-Name": "scorecard-ai", "X-Fern-SDK-Version": "0.5.2", "X-Fern-Runtime": core.RUNTIME.type, "X-Fern-Runtime-Version": core.RUNTIME.version }, (yield this._getCustomAuthorizationHeaders())),
|
|
73
73
|
contentType: "application/json",
|
|
74
74
|
timeoutMs: (requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.timeoutInSeconds) != null ? requestOptions.timeoutInSeconds * 1000 : 60000,
|
|
75
75
|
maxRetries: requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.maxRetries,
|
|
76
|
+
abortSignal: requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.abortSignal,
|
|
76
77
|
});
|
|
77
78
|
if (_response.ok) {
|
|
78
79
|
return yield serializers.traces.get.Response.parseOrThrow(_response.body, {
|
package/api/types/Grade.d.ts
CHANGED
|
@@ -12,7 +12,9 @@ export interface Grade {
|
|
|
12
12
|
binaryScore?: boolean;
|
|
13
13
|
intScore?: number;
|
|
14
14
|
reasoning?: string;
|
|
15
|
+
/** Indicates if a human should assign a grade. */
|
|
15
16
|
humanEval?: boolean;
|
|
17
|
+
/** The status of the grade. */
|
|
16
18
|
status?: Scorecard.ScoreStatus;
|
|
17
19
|
errorMessage?: string;
|
|
18
20
|
createdAt?: Date;
|
package/api/types/TestCase.d.ts
CHANGED
|
@@ -5,7 +5,9 @@ import * as Scorecard from "../index";
|
|
|
5
5
|
export interface TestCase {
|
|
6
6
|
id?: number;
|
|
7
7
|
createdAt?: Date;
|
|
8
|
+
/** The ID of the testset the testcase belongs to. */
|
|
8
9
|
testsetId: number;
|
|
10
|
+
/** The user query for the testcase. */
|
|
9
11
|
userQuery: string;
|
|
10
12
|
context?: string;
|
|
11
13
|
ideal?: string;
|
|
@@ -2,4 +2,4 @@
|
|
|
2
2
|
* This file was auto-generated by Fern from our API Definition.
|
|
3
3
|
*/
|
|
4
4
|
import * as Scorecard from "../index";
|
|
5
|
-
export declare type TestCaseCustomInputsValue =
|
|
5
|
+
export declare type TestCaseCustomInputsValue = Scorecard.FileUrl | Scorecard.JsonObject | string | number | number | boolean;
|
|
@@ -2,4 +2,4 @@
|
|
|
2
2
|
* This file was auto-generated by Fern from our API Definition.
|
|
3
3
|
*/
|
|
4
4
|
import * as Scorecard from "../index";
|
|
5
|
-
export declare type TestCaseCustomLabelsValue =
|
|
5
|
+
export declare type TestCaseCustomLabelsValue = Scorecard.FileUrl | Scorecard.JsonObject | string | number | number | boolean;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This file was auto-generated by Fern from our API Definition.
|
|
3
|
+
*/
|
|
4
|
+
export interface TestcaseDeletionResponse {
|
|
5
|
+
/** The ID of the testcase that was deleted. */
|
|
6
|
+
id: number;
|
|
7
|
+
/** The message indicating the testcase was deleted. */
|
|
8
|
+
detail: string;
|
|
9
|
+
}
|
|
@@ -12,7 +12,6 @@ export interface Testrecord {
|
|
|
12
12
|
context?: string;
|
|
13
13
|
modelResponse?: string;
|
|
14
14
|
ideal?: string;
|
|
15
|
-
fullPrompt?: string;
|
|
16
15
|
customInputs?: Record<string, Scorecard.TestrecordCustomInputsValue | undefined>;
|
|
17
16
|
customLabels?: Record<string, Scorecard.TestrecordCustomLabelsValue | undefined>;
|
|
18
17
|
customOutputs?: Record<string, Scorecard.TestrecordCustomOutputsValue | undefined>;
|
package/api/types/index.d.ts
CHANGED
|
@@ -24,6 +24,7 @@ export * from "./TestCase";
|
|
|
24
24
|
export * from "./TestCaseCreate";
|
|
25
25
|
export * from "./TestRecordCreate";
|
|
26
26
|
export * from "./TestSetCreate";
|
|
27
|
+
export * from "./TestcaseDeletionResponse";
|
|
27
28
|
export * from "./TestrecordCustomInputsValue";
|
|
28
29
|
export * from "./TestrecordCustomLabelsValue";
|
|
29
30
|
export * from "./TestrecordCustomOutputsValue";
|
package/api/types/index.js
CHANGED
|
@@ -40,6 +40,7 @@ __exportStar(require("./TestCase"), exports);
|
|
|
40
40
|
__exportStar(require("./TestCaseCreate"), exports);
|
|
41
41
|
__exportStar(require("./TestRecordCreate"), exports);
|
|
42
42
|
__exportStar(require("./TestSetCreate"), exports);
|
|
43
|
+
__exportStar(require("./TestcaseDeletionResponse"), exports);
|
|
43
44
|
__exportStar(require("./TestrecordCustomInputsValue"), exports);
|
|
44
45
|
__exportStar(require("./TestrecordCustomLabelsValue"), exports);
|
|
45
46
|
__exportStar(require("./TestrecordCustomOutputsValue"), exports);
|
|
@@ -11,6 +11,7 @@ export declare namespace Fetcher {
|
|
|
11
11
|
timeoutMs?: number;
|
|
12
12
|
maxRetries?: number;
|
|
13
13
|
withCredentials?: boolean;
|
|
14
|
+
abortSignal?: AbortSignal;
|
|
14
15
|
responseType?: "json" | "blob" | "streaming" | "text";
|
|
15
16
|
}
|
|
16
17
|
type Error = FailedStatusCodeError | NonJsonError | TimeoutError | UnknownError;
|
package/core/fetcher/Fetcher.js
CHANGED
|
@@ -63,6 +63,9 @@ function fetcherImpl(args) {
|
|
|
63
63
|
if (body instanceof Uint8Array) {
|
|
64
64
|
return body;
|
|
65
65
|
}
|
|
66
|
+
else if (args.contentType === "application/x-www-form-urlencoded" && typeof args.body === "string") {
|
|
67
|
+
return args.body;
|
|
68
|
+
}
|
|
66
69
|
else {
|
|
67
70
|
return JSON.stringify(body);
|
|
68
71
|
}
|
|
@@ -96,20 +99,27 @@ function fetcherImpl(args) {
|
|
|
96
99
|
? fetch
|
|
97
100
|
: (yield Promise.resolve().then(() => __importStar(require("node-fetch")))).default;
|
|
98
101
|
const makeRequest = () => __awaiter(this, void 0, void 0, function* () {
|
|
99
|
-
const
|
|
100
|
-
|
|
102
|
+
const signals = [];
|
|
103
|
+
// Add timeout signal
|
|
104
|
+
let timeoutAbortId = undefined;
|
|
101
105
|
if (args.timeoutMs != null) {
|
|
102
|
-
abortId =
|
|
106
|
+
const { signal, abortId } = getTimeoutSignal(args.timeoutMs);
|
|
107
|
+
timeoutAbortId = abortId;
|
|
108
|
+
signals.push(signal);
|
|
109
|
+
}
|
|
110
|
+
// Add arbitrary signal
|
|
111
|
+
if (args.abortSignal != null) {
|
|
112
|
+
signals.push(args.abortSignal);
|
|
103
113
|
}
|
|
104
114
|
const response = yield fetchFn(url, {
|
|
105
115
|
method: args.method,
|
|
106
116
|
headers,
|
|
107
117
|
body,
|
|
108
|
-
signal:
|
|
118
|
+
signal: anySignal(signals),
|
|
109
119
|
credentials: args.withCredentials ? "include" : undefined,
|
|
110
120
|
});
|
|
111
|
-
if (
|
|
112
|
-
clearTimeout(
|
|
121
|
+
if (timeoutAbortId != null) {
|
|
122
|
+
clearTimeout(timeoutAbortId);
|
|
113
123
|
}
|
|
114
124
|
return response;
|
|
115
125
|
});
|
|
@@ -175,7 +185,16 @@ function fetcherImpl(args) {
|
|
|
175
185
|
}
|
|
176
186
|
}
|
|
177
187
|
catch (error) {
|
|
178
|
-
if (
|
|
188
|
+
if (args.abortSignal != null && args.abortSignal.aborted) {
|
|
189
|
+
return {
|
|
190
|
+
ok: false,
|
|
191
|
+
error: {
|
|
192
|
+
reason: "unknown",
|
|
193
|
+
errorMessage: "The user aborted a request",
|
|
194
|
+
},
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
else if (error instanceof Error && error.name === "AbortError") {
|
|
179
198
|
return {
|
|
180
199
|
ok: false,
|
|
181
200
|
error: {
|
|
@@ -202,4 +221,36 @@ function fetcherImpl(args) {
|
|
|
202
221
|
}
|
|
203
222
|
});
|
|
204
223
|
}
|
|
224
|
+
const TIMEOUT = "timeout";
|
|
225
|
+
function getTimeoutSignal(timeoutMs) {
|
|
226
|
+
const controller = new AbortController();
|
|
227
|
+
const abortId = setTimeout(() => controller.abort(TIMEOUT), timeoutMs);
|
|
228
|
+
return { signal: controller.signal, abortId };
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* Returns an abort signal that is getting aborted when
|
|
232
|
+
* at least one of the specified abort signals is aborted.
|
|
233
|
+
*
|
|
234
|
+
* Requires at least node.js 18.
|
|
235
|
+
*/
|
|
236
|
+
function anySignal(...args) {
|
|
237
|
+
// Allowing signals to be passed either as array
|
|
238
|
+
// of signals or as multiple arguments.
|
|
239
|
+
const signals = (args.length === 1 && Array.isArray(args[0]) ? args[0] : args);
|
|
240
|
+
const controller = new AbortController();
|
|
241
|
+
for (const signal of signals) {
|
|
242
|
+
if (signal.aborted) {
|
|
243
|
+
// Exiting early if one of the signals
|
|
244
|
+
// is already aborted.
|
|
245
|
+
controller.abort(signal === null || signal === void 0 ? void 0 : signal.reason);
|
|
246
|
+
break;
|
|
247
|
+
}
|
|
248
|
+
// Listening for signals and removing the listeners
|
|
249
|
+
// when at least one symbol is aborted.
|
|
250
|
+
signal.addEventListener("abort", () => controller.abort(signal === null || signal === void 0 ? void 0 : signal.reason), {
|
|
251
|
+
signal: controller.signal,
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
return controller.signal;
|
|
255
|
+
}
|
|
205
256
|
exports.fetcher = fetcherImpl;
|
package/dist/Client.d.ts
CHANGED
|
@@ -15,10 +15,12 @@ export declare namespace ScorecardClient {
|
|
|
15
15
|
interface Options {
|
|
16
16
|
environment?: core.Supplier<environments.ScorecardEnvironment | string>;
|
|
17
17
|
apiKey: core.Supplier<string>;
|
|
18
|
+
fetcher?: core.FetchFunction;
|
|
18
19
|
}
|
|
19
20
|
interface RequestOptions {
|
|
20
21
|
timeoutInSeconds?: number;
|
|
21
22
|
maxRetries?: number;
|
|
23
|
+
abortSignal?: AbortSignal;
|
|
22
24
|
}
|
|
23
25
|
}
|
|
24
26
|
export declare class ScorecardClient {
|
|
@@ -8,17 +8,23 @@ export declare namespace Prompt {
|
|
|
8
8
|
interface Options {
|
|
9
9
|
environment?: core.Supplier<environments.ScorecardEnvironment | string>;
|
|
10
10
|
apiKey: core.Supplier<string>;
|
|
11
|
+
fetcher?: core.FetchFunction;
|
|
11
12
|
}
|
|
12
13
|
interface RequestOptions {
|
|
13
14
|
timeoutInSeconds?: number;
|
|
14
15
|
maxRetries?: number;
|
|
16
|
+
abortSignal?: AbortSignal;
|
|
15
17
|
}
|
|
16
18
|
}
|
|
17
19
|
export declare class Prompt {
|
|
18
20
|
protected readonly _options: Prompt.Options;
|
|
19
21
|
constructor(_options: Prompt.Options);
|
|
20
22
|
/**
|
|
21
|
-
*
|
|
23
|
+
* Two types of prompts can be created - a root prompt or a child prompt (aka Prompt Version in app).
|
|
24
|
+
*
|
|
25
|
+
* A root prompt can be created by providing the `name` param, and it will always be tagged as prod.
|
|
26
|
+
*
|
|
27
|
+
* A child prompt can be created by providing the `parent_id` param. Note that the `name` param in this case will be ignored as all descendents from a root prompt would share the root's name. `is_prod` can also be provided to configure whether a child should be tagged as prod.
|
|
22
28
|
*
|
|
23
29
|
* @param {Scorecard.PromptCreateParams} request
|
|
24
30
|
* @param {Prompt.RequestOptions} requestOptions - Request-specific configuration.
|
|
@@ -30,7 +36,7 @@ export declare class Prompt {
|
|
|
30
36
|
*
|
|
31
37
|
* @example
|
|
32
38
|
* await scorecard.prompt.create({
|
|
33
|
-
* promptTemplate: "
|
|
39
|
+
* promptTemplate: "<system>\nYou are a helpful assistant. Use the provided context to answer the user's query.\n\nContext: {context}\n</system>\n\n<user>\n{user_query}\n</user>",
|
|
34
40
|
* name: "Prompt Name",
|
|
35
41
|
* description: "Description of the prompt",
|
|
36
42
|
* modelParams: {
|
|
@@ -43,7 +49,7 @@ export declare class Prompt {
|
|
|
43
49
|
*
|
|
44
50
|
* @example
|
|
45
51
|
* await scorecard.prompt.create({
|
|
46
|
-
* promptTemplate: "
|
|
52
|
+
* promptTemplate: "<system>\nYou are a helpful assistant. Use the provided context to answer the user's query.\n\nContext: {context}\n</system>\n\n<user>\n{user_query}\n</user>",
|
|
47
53
|
* parentId: "7ac3cbd5-3b99-4e72-97f3-9cd2e749cace",
|
|
48
54
|
* description: "Description of the prompt",
|
|
49
55
|
* modelParams: {
|
|
@@ -57,35 +63,24 @@ export declare class Prompt {
|
|
|
57
63
|
*/
|
|
58
64
|
create(request: Scorecard.PromptCreateParams, requestOptions?: Prompt.RequestOptions): Promise<Scorecard.Prompt>;
|
|
59
65
|
/**
|
|
60
|
-
* Retrieve
|
|
61
|
-
*
|
|
62
|
-
* @param {Prompt.RequestOptions} requestOptions - Request-specific configuration.
|
|
63
|
-
*
|
|
64
|
-
* @throws {@link Scorecard.UnauthorizedError}
|
|
65
|
-
* @throws {@link Scorecard.ForbiddenError}
|
|
66
|
-
* @throws {@link Scorecard.NotFoundError}
|
|
67
|
-
*
|
|
68
|
-
* @example
|
|
69
|
-
* await scorecard.prompt.getRoots()
|
|
70
|
-
*/
|
|
71
|
-
getRoots(requestOptions?: Prompt.RequestOptions): Promise<Scorecard.Prompt[]>;
|
|
72
|
-
/**
|
|
73
|
-
* Retrieve prod prompts for the org
|
|
66
|
+
* Retrieve a prompt by id
|
|
74
67
|
*
|
|
68
|
+
* @param {string} id - The id of the prompt to get.
|
|
75
69
|
* @param {Prompt.RequestOptions} requestOptions - Request-specific configuration.
|
|
76
70
|
*
|
|
77
71
|
* @throws {@link Scorecard.UnauthorizedError}
|
|
78
72
|
* @throws {@link Scorecard.ForbiddenError}
|
|
79
73
|
* @throws {@link Scorecard.NotFoundError}
|
|
74
|
+
* @throws {@link Scorecard.UnprocessableEntityError}
|
|
80
75
|
*
|
|
81
76
|
* @example
|
|
82
|
-
* await scorecard.prompt.
|
|
77
|
+
* await scorecard.prompt.get("id")
|
|
83
78
|
*/
|
|
84
|
-
|
|
79
|
+
get(id: string, requestOptions?: Prompt.RequestOptions): Promise<Scorecard.Prompt>;
|
|
85
80
|
/**
|
|
86
|
-
*
|
|
81
|
+
* Delete a root prompt and all of its children.
|
|
87
82
|
*
|
|
88
|
-
* @param {string} id - The id of the prompt to
|
|
83
|
+
* @param {string} id - The id of the root prompt to delete.
|
|
89
84
|
* @param {Prompt.RequestOptions} requestOptions - Request-specific configuration.
|
|
90
85
|
*
|
|
91
86
|
* @throws {@link Scorecard.UnauthorizedError}
|
|
@@ -94,11 +89,13 @@ export declare class Prompt {
|
|
|
94
89
|
* @throws {@link Scorecard.UnprocessableEntityError}
|
|
95
90
|
*
|
|
96
91
|
* @example
|
|
97
|
-
* await scorecard.prompt.
|
|
92
|
+
* await scorecard.prompt.delete("id")
|
|
98
93
|
*/
|
|
99
|
-
|
|
94
|
+
delete(id: string, requestOptions?: Prompt.RequestOptions): Promise<unknown>;
|
|
100
95
|
/**
|
|
101
|
-
* Update a prompt
|
|
96
|
+
* Update a prompt.
|
|
97
|
+
*
|
|
98
|
+
* `is_prod` tags the provided prompt as the production prompt within the prompt graph.
|
|
102
99
|
*
|
|
103
100
|
* @param {string} id - The id of the prompt to update.
|
|
104
101
|
* @param {Scorecard.PromptUpdateParams} request
|
|
@@ -111,7 +108,7 @@ export declare class Prompt {
|
|
|
111
108
|
*
|
|
112
109
|
* @example
|
|
113
110
|
* await scorecard.prompt.update("id", {
|
|
114
|
-
*
|
|
111
|
+
* isProd: true
|
|
115
112
|
* })
|
|
116
113
|
*/
|
|
117
114
|
update(id: string, request?: Scorecard.PromptUpdateParams, requestOptions?: Prompt.RequestOptions): Promise<Scorecard.Prompt>;
|
|
@@ -130,21 +127,6 @@ export declare class Prompt {
|
|
|
130
127
|
* await scorecard.prompt.getByName("name")
|
|
131
128
|
*/
|
|
132
129
|
getByName(name: string, requestOptions?: Prompt.RequestOptions): Promise<Scorecard.Prompt>;
|
|
133
|
-
/**
|
|
134
|
-
* Retrieve all nodes within the same graph as the prompt, sorted by created_at desc
|
|
135
|
-
*
|
|
136
|
-
* @param {string} id - The id of the prompt.
|
|
137
|
-
* @param {Prompt.RequestOptions} requestOptions - Request-specific configuration.
|
|
138
|
-
*
|
|
139
|
-
* @throws {@link Scorecard.UnauthorizedError}
|
|
140
|
-
* @throws {@link Scorecard.ForbiddenError}
|
|
141
|
-
* @throws {@link Scorecard.NotFoundError}
|
|
142
|
-
* @throws {@link Scorecard.UnprocessableEntityError}
|
|
143
|
-
*
|
|
144
|
-
* @example
|
|
145
|
-
* await scorecard.prompt.getGraph("id")
|
|
146
|
-
*/
|
|
147
|
-
getGraph(id: string, requestOptions?: Prompt.RequestOptions): Promise<Scorecard.Prompt[]>;
|
|
148
130
|
protected _getCustomAuthorizationHeaders(): Promise<{
|
|
149
131
|
"X-API-Key": string;
|
|
150
132
|
}>;
|