scorecard-ai-mcp 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/docs-search-tool.d.mts +44 -0
- package/docs-search-tool.d.mts.map +1 -0
- package/docs-search-tool.d.ts +44 -0
- package/docs-search-tool.d.ts.map +1 -0
- package/docs-search-tool.js +43 -0
- package/docs-search-tool.js.map +1 -0
- package/docs-search-tool.mjs +39 -0
- package/docs-search-tool.mjs.map +1 -0
- package/options.d.mts +1 -0
- package/options.d.mts.map +1 -1
- package/options.d.ts +1 -0
- package/options.d.ts.map +1 -1
- package/options.js +13 -6
- package/options.js.map +1 -1
- package/options.mjs +13 -6
- package/options.mjs.map +1 -1
- package/package.json +13 -2
- package/server.d.mts.map +1 -1
- package/server.d.ts.map +1 -1
- package/server.js +8 -4
- package/server.js.map +1 -1
- package/server.mjs +8 -4
- package/server.mjs.map +1 -1
- package/src/docs-search-tool.ts +48 -0
- package/src/options.ts +17 -7
- package/src/server.ts +8 -5
- package/src/tools/index.ts +10 -0
- package/src/tools/metrics/create-metrics.ts +120 -2
- package/src/tools/metrics/get-metrics.ts +39 -0
- package/src/tools/metrics/list-metrics.ts +51 -0
- package/src/tools/metrics/update-metrics.ts +120 -2
- package/src/tools/records/list-records.ts +58 -0
- package/src/tools/runs/create-runs.ts +1 -1
- package/src/tools/runs/get-runs.ts +47 -0
- package/src/tools/runs/list-runs.ts +58 -0
- package/src/tools/systems/delete-systems.ts +1 -1
- package/src/tools/testcases/create-testcases.ts +1 -1
- package/src/tools/testcases/delete-testcases.ts +1 -1
- package/src/tools/testsets/delete-testsets.ts +1 -1
- package/tools/index.d.mts.map +1 -1
- package/tools/index.d.ts.map +1 -1
- package/tools/index.js +10 -0
- package/tools/index.js.map +1 -1
- package/tools/index.mjs +10 -0
- package/tools/index.mjs.map +1 -1
- package/tools/metrics/create-metrics.d.mts.map +1 -1
- package/tools/metrics/create-metrics.d.ts.map +1 -1
- package/tools/metrics/create-metrics.js +119 -2
- package/tools/metrics/create-metrics.js.map +1 -1
- package/tools/metrics/create-metrics.mjs +119 -2
- package/tools/metrics/create-metrics.mjs.map +1 -1
- package/tools/metrics/get-metrics.d.mts +45 -0
- package/tools/metrics/get-metrics.d.mts.map +1 -0
- package/tools/metrics/get-metrics.d.ts +45 -0
- package/tools/metrics/get-metrics.d.ts.map +1 -0
- package/tools/metrics/get-metrics.js +36 -0
- package/tools/metrics/get-metrics.js.map +1 -0
- package/tools/metrics/get-metrics.mjs +32 -0
- package/tools/metrics/get-metrics.mjs.map +1 -0
- package/tools/metrics/list-metrics.d.mts +45 -0
- package/tools/metrics/list-metrics.d.mts.map +1 -0
- package/tools/metrics/list-metrics.d.ts +45 -0
- package/tools/metrics/list-metrics.d.ts.map +1 -0
- package/tools/metrics/list-metrics.js +45 -0
- package/tools/metrics/list-metrics.js.map +1 -0
- package/tools/metrics/list-metrics.mjs +41 -0
- package/tools/metrics/list-metrics.mjs.map +1 -0
- package/tools/metrics/update-metrics.d.mts.map +1 -1
- package/tools/metrics/update-metrics.d.ts.map +1 -1
- package/tools/metrics/update-metrics.js +119 -2
- package/tools/metrics/update-metrics.js.map +1 -1
- package/tools/metrics/update-metrics.mjs +119 -2
- package/tools/metrics/update-metrics.mjs.map +1 -1
- package/tools/records/list-records.d.mts +45 -0
- package/tools/records/list-records.d.mts.map +1 -0
- package/tools/records/list-records.d.ts +45 -0
- package/tools/records/list-records.d.ts.map +1 -0
- package/tools/records/list-records.js +51 -0
- package/tools/records/list-records.js.map +1 -0
- package/tools/records/list-records.mjs +47 -0
- package/tools/records/list-records.mjs.map +1 -0
- package/tools/runs/create-runs.js +1 -1
- package/tools/runs/create-runs.js.map +1 -1
- package/tools/runs/create-runs.mjs +1 -1
- package/tools/runs/create-runs.mjs.map +1 -1
- package/tools/runs/get-runs.d.mts +45 -0
- package/tools/runs/get-runs.d.mts.map +1 -0
- package/tools/runs/get-runs.d.ts +45 -0
- package/tools/runs/get-runs.d.ts.map +1 -0
- package/tools/runs/get-runs.js +42 -0
- package/tools/runs/get-runs.js.map +1 -0
- package/tools/runs/get-runs.mjs +38 -0
- package/tools/runs/get-runs.mjs.map +1 -0
- package/tools/runs/list-runs.d.mts +45 -0
- package/tools/runs/list-runs.d.mts.map +1 -0
- package/tools/runs/list-runs.d.ts +45 -0
- package/tools/runs/list-runs.d.ts.map +1 -0
- package/tools/runs/list-runs.js +51 -0
- package/tools/runs/list-runs.js.map +1 -0
- package/tools/runs/list-runs.mjs +47 -0
- package/tools/runs/list-runs.mjs.map +1 -0
- package/tools/systems/delete-systems.js +1 -1
- package/tools/systems/delete-systems.js.map +1 -1
- package/tools/systems/delete-systems.mjs +1 -1
- package/tools/systems/delete-systems.mjs.map +1 -1
- package/tools/testcases/create-testcases.js +1 -1
- package/tools/testcases/create-testcases.js.map +1 -1
- package/tools/testcases/create-testcases.mjs +1 -1
- package/tools/testcases/create-testcases.mjs.map +1 -1
- package/tools/testcases/delete-testcases.js +1 -1
- package/tools/testcases/delete-testcases.js.map +1 -1
- package/tools/testcases/delete-testcases.mjs +1 -1
- package/tools/testcases/delete-testcases.mjs.map +1 -1
- package/tools/testsets/delete-testsets.js +1 -1
- package/tools/testsets/delete-testsets.js.map +1 -1
- package/tools/testsets/delete-testsets.mjs +1 -1
- package/tools/testsets/delete-testsets.mjs.map +1 -1
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.handler = exports.tool = exports.metadata = void 0;
|
|
5
|
+
const types_1 = require("scorecard-ai-mcp/tools/types");
|
|
6
|
+
exports.metadata = {
|
|
7
|
+
resource: 'metrics',
|
|
8
|
+
operation: 'read',
|
|
9
|
+
tags: [],
|
|
10
|
+
httpMethod: 'get',
|
|
11
|
+
httpPath: '/projects/{projectId}/metrics',
|
|
12
|
+
operationId: 'listMetrics',
|
|
13
|
+
};
|
|
14
|
+
exports.tool = {
|
|
15
|
+
name: 'list_metrics',
|
|
16
|
+
description: 'List Metrics configured for the specified Project. Metrics are returned in reverse chronological order.',
|
|
17
|
+
inputSchema: {
|
|
18
|
+
type: 'object',
|
|
19
|
+
properties: {
|
|
20
|
+
projectId: {
|
|
21
|
+
type: 'string',
|
|
22
|
+
},
|
|
23
|
+
cursor: {
|
|
24
|
+
type: 'string',
|
|
25
|
+
description: 'Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of results.',
|
|
26
|
+
},
|
|
27
|
+
limit: {
|
|
28
|
+
type: 'integer',
|
|
29
|
+
description: 'Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.',
|
|
30
|
+
},
|
|
31
|
+
},
|
|
32
|
+
required: ['projectId'],
|
|
33
|
+
},
|
|
34
|
+
annotations: {
|
|
35
|
+
readOnlyHint: true,
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
const handler = async (client, args) => {
|
|
39
|
+
const { projectId, ...body } = args;
|
|
40
|
+
const response = await client.metrics.list(projectId, body).asResponse();
|
|
41
|
+
return (0, types_1.asTextContentResult)(await response.json());
|
|
42
|
+
};
|
|
43
|
+
exports.handler = handler;
|
|
44
|
+
exports.default = { metadata: exports.metadata, tool: exports.tool, handler: exports.handler };
|
|
45
|
+
//# sourceMappingURL=list-metrics.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"list-metrics.js","sourceRoot":"","sources":["../../src/tools/metrics/list-metrics.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,wDAA6E;AAKhE,QAAA,QAAQ,GAAa;IAChC,QAAQ,EAAE,SAAS;IACnB,SAAS,EAAE,MAAM;IACjB,IAAI,EAAE,EAAE;IACR,UAAU,EAAE,KAAK;IACjB,QAAQ,EAAE,+BAA+B;IACzC,WAAW,EAAE,aAAa;CAC3B,CAAC;AAEW,QAAA,IAAI,GAAS;IACxB,IAAI,EAAE,cAAc;IACpB,WAAW,EACT,yGAAyG;IAC3G,WAAW,EAAE;QACX,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;aACf;YACD,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,WAAW,EACT,0GAA0G;aAC7G;YACD,KAAK,EAAE;gBACL,IAAI,EAAE,SAAS;gBACf,WAAW,EACT,iGAAiG;aACpG;SACF;QACD,QAAQ,EAAE,CAAC,WAAW,CAAC;KACxB;IACD,WAAW,EAAE;QACX,YAAY,EAAE,IAAI;KACnB;CACF,CAAC;AAEK,MAAM,OAAO,GAAG,KAAK,EAAE,MAAiB,EAAE,IAAyC,EAAE,EAAE;IAC5F,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,GAAG,IAAW,CAAC;IAC3C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;IACzE,OAAO,IAAA,2BAAmB,EAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;AACpD,CAAC,CAAC;AAJW,QAAA,OAAO,WAIlB;AAEF,kBAAe,EAAE,QAAQ,EAAR,gBAAQ,EAAE,IAAI,EAAJ,YAAI,EAAE,OAAO,EAAP,eAAO,EAAE,CAAC"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
2
|
+
import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
|
|
3
|
+
export const metadata = {
|
|
4
|
+
resource: 'metrics',
|
|
5
|
+
operation: 'read',
|
|
6
|
+
tags: [],
|
|
7
|
+
httpMethod: 'get',
|
|
8
|
+
httpPath: '/projects/{projectId}/metrics',
|
|
9
|
+
operationId: 'listMetrics',
|
|
10
|
+
};
|
|
11
|
+
export const tool = {
|
|
12
|
+
name: 'list_metrics',
|
|
13
|
+
description: 'List Metrics configured for the specified Project. Metrics are returned in reverse chronological order.',
|
|
14
|
+
inputSchema: {
|
|
15
|
+
type: 'object',
|
|
16
|
+
properties: {
|
|
17
|
+
projectId: {
|
|
18
|
+
type: 'string',
|
|
19
|
+
},
|
|
20
|
+
cursor: {
|
|
21
|
+
type: 'string',
|
|
22
|
+
description: 'Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of results.',
|
|
23
|
+
},
|
|
24
|
+
limit: {
|
|
25
|
+
type: 'integer',
|
|
26
|
+
description: 'Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.',
|
|
27
|
+
},
|
|
28
|
+
},
|
|
29
|
+
required: ['projectId'],
|
|
30
|
+
},
|
|
31
|
+
annotations: {
|
|
32
|
+
readOnlyHint: true,
|
|
33
|
+
},
|
|
34
|
+
};
|
|
35
|
+
export const handler = async (client, args) => {
|
|
36
|
+
const { projectId, ...body } = args;
|
|
37
|
+
const response = await client.metrics.list(projectId, body).asResponse();
|
|
38
|
+
return asTextContentResult(await response.json());
|
|
39
|
+
};
|
|
40
|
+
export default { metadata, tool, handler };
|
|
41
|
+
//# sourceMappingURL=list-metrics.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"list-metrics.mjs","sourceRoot":"","sources":["../../src/tools/metrics/list-metrics.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAY,mBAAmB,EAAE,MAAM,8BAA8B;AAK5E,MAAM,CAAC,MAAM,QAAQ,GAAa;IAChC,QAAQ,EAAE,SAAS;IACnB,SAAS,EAAE,MAAM;IACjB,IAAI,EAAE,EAAE;IACR,UAAU,EAAE,KAAK;IACjB,QAAQ,EAAE,+BAA+B;IACzC,WAAW,EAAE,aAAa;CAC3B,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAAS;IACxB,IAAI,EAAE,cAAc;IACpB,WAAW,EACT,yGAAyG;IAC3G,WAAW,EAAE;QACX,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;aACf;YACD,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,WAAW,EACT,0GAA0G;aAC7G;YACD,KAAK,EAAE;gBACL,IAAI,EAAE,SAAS;gBACf,WAAW,EACT,iGAAiG;aACpG;SACF;QACD,QAAQ,EAAE,CAAC,WAAW,CAAC;KACxB;IACD,WAAW,EAAE;QACX,YAAY,EAAE,IAAI;KACnB;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAAE,MAAiB,EAAE,IAAyC,EAAE,EAAE;IAC5F,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,GAAG,IAAW,CAAC;IAC3C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;IACzE,OAAO,mBAAmB,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;AACpD,CAAC,CAAC;AAEF,eAAe,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"update-metrics.d.mts","sourceRoot":"","sources":["../../src/tools/metrics/update-metrics.ts"],"names":[],"mappings":"OAEO,EAAE,QAAQ,EAAuB,MAAM,8BAA8B;OAErE,EAAE,IAAI,EAAE,MAAM,oCAAoC;OAClD,SAAS,MAAM,cAAc;AAEpC,eAAO,MAAM,QAAQ,EAAE,QAOtB,CAAC;AAEF,eAAO,MAAM,IAAI,EAAE,
|
|
1
|
+
{"version":3,"file":"update-metrics.d.mts","sourceRoot":"","sources":["../../src/tools/metrics/update-metrics.ts"],"names":[],"mappings":"OAEO,EAAE,QAAQ,EAAuB,MAAM,8BAA8B;OAErE,EAAE,IAAI,EAAE,MAAM,oCAAoC;OAClD,SAAS,MAAM,cAAc;AAEpC,eAAO,MAAM,QAAQ,EAAE,QAOtB,CAAC;AAEF,eAAO,MAAM,IAAI,EAAE,IAgWlB,CAAC;AAEF,eAAO,MAAM,OAAO,GAAU,QAAQ,SAAS,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,mEAGzF,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;sBAHoC,SAAS,QAAQ,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;;AAK1F,wBAA2C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"update-metrics.d.ts","sourceRoot":"","sources":["../../src/tools/metrics/update-metrics.ts"],"names":[],"mappings":"OAEO,EAAE,QAAQ,EAAuB,MAAM,8BAA8B;OAErE,EAAE,IAAI,EAAE,MAAM,oCAAoC;OAClD,SAAS,MAAM,cAAc;AAEpC,eAAO,MAAM,QAAQ,EAAE,QAOtB,CAAC;AAEF,eAAO,MAAM,IAAI,EAAE,
|
|
1
|
+
{"version":3,"file":"update-metrics.d.ts","sourceRoot":"","sources":["../../src/tools/metrics/update-metrics.ts"],"names":[],"mappings":"OAEO,EAAE,QAAQ,EAAuB,MAAM,8BAA8B;OAErE,EAAE,IAAI,EAAE,MAAM,oCAAoC;OAClD,SAAS,MAAM,cAAc;AAEpC,eAAO,MAAM,QAAQ,EAAE,QAOtB,CAAC;AAEF,eAAO,MAAM,IAAI,EAAE,IAgWlB,CAAC;AAEF,eAAO,MAAM,OAAO,GAAU,QAAQ,SAAS,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,mEAGzF,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;sBAHoC,SAAS,QAAQ,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;;AAK1F,wBAA2C"}
|
|
@@ -121,7 +121,7 @@ exports.tool = {
|
|
|
121
121
|
},
|
|
122
122
|
guidelines: {
|
|
123
123
|
type: 'string',
|
|
124
|
-
description: '
|
|
124
|
+
description: 'Guidelines for heuristic evaluation logic.',
|
|
125
125
|
},
|
|
126
126
|
name: {
|
|
127
127
|
type: 'string',
|
|
@@ -134,6 +134,123 @@ exports.tool = {
|
|
|
134
134
|
},
|
|
135
135
|
required: ['metricId', 'evalType', 'outputType'],
|
|
136
136
|
},
|
|
137
|
+
{
|
|
138
|
+
type: 'object',
|
|
139
|
+
properties: {
|
|
140
|
+
metricId: {
|
|
141
|
+
type: 'string',
|
|
142
|
+
},
|
|
143
|
+
evalType: {
|
|
144
|
+
type: 'string',
|
|
145
|
+
description: 'AI-based evaluation type.',
|
|
146
|
+
enum: ['ai'],
|
|
147
|
+
},
|
|
148
|
+
outputType: {
|
|
149
|
+
type: 'string',
|
|
150
|
+
description: 'Float output type (0-1).',
|
|
151
|
+
enum: ['float'],
|
|
152
|
+
},
|
|
153
|
+
description: {
|
|
154
|
+
type: 'string',
|
|
155
|
+
description: 'The description of the Metric.',
|
|
156
|
+
},
|
|
157
|
+
evalModelName: {
|
|
158
|
+
type: 'string',
|
|
159
|
+
description: 'The AI model to use for evaluation.',
|
|
160
|
+
},
|
|
161
|
+
guidelines: {
|
|
162
|
+
type: 'string',
|
|
163
|
+
description: 'Guidelines for AI evaluation on how to score the metric.',
|
|
164
|
+
},
|
|
165
|
+
name: {
|
|
166
|
+
type: 'string',
|
|
167
|
+
description: 'The name of the Metric.',
|
|
168
|
+
},
|
|
169
|
+
passingThreshold: {
|
|
170
|
+
type: 'number',
|
|
171
|
+
description: 'Threshold for determining pass/fail from float scores (0.0-1.0).',
|
|
172
|
+
},
|
|
173
|
+
promptTemplate: {
|
|
174
|
+
type: 'string',
|
|
175
|
+
description: 'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
|
|
176
|
+
},
|
|
177
|
+
temperature: {
|
|
178
|
+
type: 'number',
|
|
179
|
+
description: 'The temperature for AI evaluation (0-2).',
|
|
180
|
+
},
|
|
181
|
+
},
|
|
182
|
+
required: ['metricId', 'evalType', 'outputType'],
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
type: 'object',
|
|
186
|
+
properties: {
|
|
187
|
+
metricId: {
|
|
188
|
+
type: 'string',
|
|
189
|
+
},
|
|
190
|
+
evalType: {
|
|
191
|
+
type: 'string',
|
|
192
|
+
description: 'Human-based evaluation type.',
|
|
193
|
+
enum: ['human'],
|
|
194
|
+
},
|
|
195
|
+
outputType: {
|
|
196
|
+
type: 'string',
|
|
197
|
+
description: 'Float output type (0-1).',
|
|
198
|
+
enum: ['float'],
|
|
199
|
+
},
|
|
200
|
+
description: {
|
|
201
|
+
type: 'string',
|
|
202
|
+
description: 'The description of the Metric.',
|
|
203
|
+
},
|
|
204
|
+
guidelines: {
|
|
205
|
+
type: 'string',
|
|
206
|
+
description: 'Guidelines for human evaluators.',
|
|
207
|
+
},
|
|
208
|
+
name: {
|
|
209
|
+
type: 'string',
|
|
210
|
+
description: 'The name of the Metric.',
|
|
211
|
+
},
|
|
212
|
+
passingThreshold: {
|
|
213
|
+
type: 'number',
|
|
214
|
+
description: 'Threshold for determining pass/fail from float scores (0.0-1.0).',
|
|
215
|
+
},
|
|
216
|
+
},
|
|
217
|
+
required: ['metricId', 'evalType', 'outputType'],
|
|
218
|
+
},
|
|
219
|
+
{
|
|
220
|
+
type: 'object',
|
|
221
|
+
properties: {
|
|
222
|
+
metricId: {
|
|
223
|
+
type: 'string',
|
|
224
|
+
},
|
|
225
|
+
evalType: {
|
|
226
|
+
type: 'string',
|
|
227
|
+
description: 'Heuristic-based evaluation type.',
|
|
228
|
+
enum: ['heuristic'],
|
|
229
|
+
},
|
|
230
|
+
outputType: {
|
|
231
|
+
type: 'string',
|
|
232
|
+
description: 'Float output type (0-1).',
|
|
233
|
+
enum: ['float'],
|
|
234
|
+
},
|
|
235
|
+
description: {
|
|
236
|
+
type: 'string',
|
|
237
|
+
description: 'The description of the Metric.',
|
|
238
|
+
},
|
|
239
|
+
guidelines: {
|
|
240
|
+
type: 'string',
|
|
241
|
+
description: 'Guidelines for heuristic evaluation logic.',
|
|
242
|
+
},
|
|
243
|
+
name: {
|
|
244
|
+
type: 'string',
|
|
245
|
+
description: 'The name of the Metric.',
|
|
246
|
+
},
|
|
247
|
+
passingThreshold: {
|
|
248
|
+
type: 'number',
|
|
249
|
+
description: 'Threshold for determining pass/fail from float scores (0.0-1.0).',
|
|
250
|
+
},
|
|
251
|
+
},
|
|
252
|
+
required: ['metricId', 'evalType', 'outputType'],
|
|
253
|
+
},
|
|
137
254
|
{
|
|
138
255
|
type: 'object',
|
|
139
256
|
properties: {
|
|
@@ -230,7 +347,7 @@ exports.tool = {
|
|
|
230
347
|
},
|
|
231
348
|
guidelines: {
|
|
232
349
|
type: 'string',
|
|
233
|
-
description: '
|
|
350
|
+
description: 'Guidelines for heuristic evaluation logic.',
|
|
234
351
|
},
|
|
235
352
|
name: {
|
|
236
353
|
type: 'string',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"update-metrics.js","sourceRoot":"","sources":["../../src/tools/metrics/update-metrics.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,wDAA6E;AAKhE,QAAA,QAAQ,GAAa;IAChC,QAAQ,EAAE,SAAS;IACnB,SAAS,EAAE,OAAO;IAClB,IAAI,EAAE,EAAE;IACR,UAAU,EAAE,OAAO;IACnB,QAAQ,EAAE,qBAAqB;IAC/B,WAAW,EAAE,cAAc;CAC5B,CAAC;AAEW,QAAA,IAAI,GAAS;IACxB,IAAI,EAAE,gBAAgB;IACtB,WAAW,EACT,wKAAwK;IAC1K,WAAW,EAAE;QACX,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE;YACL;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,2BAA2B;wBACxC,IAAI,EAAE,CAAC,IAAI,CAAC;qBACb;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,aAAa,EAAE;wBACb,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,qCAAqC;qBACnD;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0DAA0D;qBACxE;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,oEAAoE;qBAClF;oBACD,cAAc,EAAE;wBACd,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,kGAAkG;qBACrG;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0CAA0C;qBACxD;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,8BAA8B;wBAC3C,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;qBAChD;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,oEAAoE;qBAClF;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;wBAC/C,IAAI,EAAE,CAAC,WAAW,CAAC;qBACpB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,
|
|
1
|
+
{"version":3,"file":"update-metrics.js","sourceRoot":"","sources":["../../src/tools/metrics/update-metrics.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,wDAA6E;AAKhE,QAAA,QAAQ,GAAa;IAChC,QAAQ,EAAE,SAAS;IACnB,SAAS,EAAE,OAAO;IAClB,IAAI,EAAE,EAAE;IACR,UAAU,EAAE,OAAO;IACnB,QAAQ,EAAE,qBAAqB;IAC/B,WAAW,EAAE,cAAc;CAC5B,CAAC;AAEW,QAAA,IAAI,GAAS;IACxB,IAAI,EAAE,gBAAgB;IACtB,WAAW,EACT,wKAAwK;IAC1K,WAAW,EAAE;QACX,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE;YACL;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,2BAA2B;wBACxC,IAAI,EAAE,CAAC,IAAI,CAAC;qBACb;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,aAAa,EAAE;wBACb,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,qCAAqC;qBACnD;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0DAA0D;qBACxE;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,oEAAoE;qBAClF;oBACD,cAAc,EAAE;wBACd,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,kGAAkG;qBACrG;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0CAA0C;qBACxD;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,8BAA8B;wBAC3C,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;qBAChD;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,oEAAoE;qBAClF;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;wBAC/C,IAAI,EAAE,CAAC,WAAW,CAAC;qBACpB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,4CAA4C;qBAC1D;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,oEAAoE;qBAClF;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,2BAA2B;wBACxC,IAAI,EAAE,CAAC,IAAI,CAAC;qBACb;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0BAA0B;wBACvC,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,aAAa,EAAE;wBACb,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,qCAAqC;qBACnD;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0DAA0D;qBACxE;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kEAAkE;qBAChF;oBACD,cAAc,EAAE;wBACd,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,kGAAkG;qBACrG;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0CAA0C;qBACxD;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,8BAA8B;wBAC3C,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0BAA0B;wBACvC,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;qBAChD;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kEAAkE;qBAChF;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;wBAC/C,IAAI,EAAE,CAAC,WAAW,CAAC;qBACpB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0BAA0B;wBACvC,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,4CAA4C;qBAC1D;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kEAAkE;qBAChF;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,2BAA2B;wBACxC,IAAI,EAAE,CAAC,IAAI,CAAC;qBACb;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,SAAS,CAAC;qBAClB;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,aAAa,EAAE;wBACb,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,qCAAqC;qBACnD;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0DAA0D;qBACxE;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,cAAc,EAAE;wBACd,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,kGAAkG;qBACrG;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0CAA0C;qBACxD;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,8BAA8B;wBAC3C,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,SAAS,CAAC;qBAClB;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;qBAChD;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;wBAC/C,IAAI,EAAE,CAAC,WAAW,CAAC;qBACpB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,SAAS,CAAC;qBAClB;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,4CAA4C;qBAC1D;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;SACF;KACF;IACD,WAAW,EAAE,EAAE;CAChB,CAAC;AAEK,MAAM,OAAO,GAAG,KAAK,EAAE,MAAiB,EAAE,IAAyC,EAAE,EAAE;IAC5F,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,IAAW,CAAC;IAC1C,OAAO,IAAA,2BAAmB,EAAC,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;AAC1E,CAAC,CAAC;AAHW,QAAA,OAAO,WAGlB;AAEF,kBAAe,EAAE,QAAQ,EAAR,gBAAQ,EAAE,IAAI,EAAJ,YAAI,EAAE,OAAO,EAAP,eAAO,EAAE,CAAC"}
|
|
@@ -118,7 +118,7 @@ export const tool = {
|
|
|
118
118
|
},
|
|
119
119
|
guidelines: {
|
|
120
120
|
type: 'string',
|
|
121
|
-
description: '
|
|
121
|
+
description: 'Guidelines for heuristic evaluation logic.',
|
|
122
122
|
},
|
|
123
123
|
name: {
|
|
124
124
|
type: 'string',
|
|
@@ -131,6 +131,123 @@ export const tool = {
|
|
|
131
131
|
},
|
|
132
132
|
required: ['metricId', 'evalType', 'outputType'],
|
|
133
133
|
},
|
|
134
|
+
{
|
|
135
|
+
type: 'object',
|
|
136
|
+
properties: {
|
|
137
|
+
metricId: {
|
|
138
|
+
type: 'string',
|
|
139
|
+
},
|
|
140
|
+
evalType: {
|
|
141
|
+
type: 'string',
|
|
142
|
+
description: 'AI-based evaluation type.',
|
|
143
|
+
enum: ['ai'],
|
|
144
|
+
},
|
|
145
|
+
outputType: {
|
|
146
|
+
type: 'string',
|
|
147
|
+
description: 'Float output type (0-1).',
|
|
148
|
+
enum: ['float'],
|
|
149
|
+
},
|
|
150
|
+
description: {
|
|
151
|
+
type: 'string',
|
|
152
|
+
description: 'The description of the Metric.',
|
|
153
|
+
},
|
|
154
|
+
evalModelName: {
|
|
155
|
+
type: 'string',
|
|
156
|
+
description: 'The AI model to use for evaluation.',
|
|
157
|
+
},
|
|
158
|
+
guidelines: {
|
|
159
|
+
type: 'string',
|
|
160
|
+
description: 'Guidelines for AI evaluation on how to score the metric.',
|
|
161
|
+
},
|
|
162
|
+
name: {
|
|
163
|
+
type: 'string',
|
|
164
|
+
description: 'The name of the Metric.',
|
|
165
|
+
},
|
|
166
|
+
passingThreshold: {
|
|
167
|
+
type: 'number',
|
|
168
|
+
description: 'Threshold for determining pass/fail from float scores (0.0-1.0).',
|
|
169
|
+
},
|
|
170
|
+
promptTemplate: {
|
|
171
|
+
type: 'string',
|
|
172
|
+
description: 'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
|
|
173
|
+
},
|
|
174
|
+
temperature: {
|
|
175
|
+
type: 'number',
|
|
176
|
+
description: 'The temperature for AI evaluation (0-2).',
|
|
177
|
+
},
|
|
178
|
+
},
|
|
179
|
+
required: ['metricId', 'evalType', 'outputType'],
|
|
180
|
+
},
|
|
181
|
+
{
|
|
182
|
+
type: 'object',
|
|
183
|
+
properties: {
|
|
184
|
+
metricId: {
|
|
185
|
+
type: 'string',
|
|
186
|
+
},
|
|
187
|
+
evalType: {
|
|
188
|
+
type: 'string',
|
|
189
|
+
description: 'Human-based evaluation type.',
|
|
190
|
+
enum: ['human'],
|
|
191
|
+
},
|
|
192
|
+
outputType: {
|
|
193
|
+
type: 'string',
|
|
194
|
+
description: 'Float output type (0-1).',
|
|
195
|
+
enum: ['float'],
|
|
196
|
+
},
|
|
197
|
+
description: {
|
|
198
|
+
type: 'string',
|
|
199
|
+
description: 'The description of the Metric.',
|
|
200
|
+
},
|
|
201
|
+
guidelines: {
|
|
202
|
+
type: 'string',
|
|
203
|
+
description: 'Guidelines for human evaluators.',
|
|
204
|
+
},
|
|
205
|
+
name: {
|
|
206
|
+
type: 'string',
|
|
207
|
+
description: 'The name of the Metric.',
|
|
208
|
+
},
|
|
209
|
+
passingThreshold: {
|
|
210
|
+
type: 'number',
|
|
211
|
+
description: 'Threshold for determining pass/fail from float scores (0.0-1.0).',
|
|
212
|
+
},
|
|
213
|
+
},
|
|
214
|
+
required: ['metricId', 'evalType', 'outputType'],
|
|
215
|
+
},
|
|
216
|
+
{
|
|
217
|
+
type: 'object',
|
|
218
|
+
properties: {
|
|
219
|
+
metricId: {
|
|
220
|
+
type: 'string',
|
|
221
|
+
},
|
|
222
|
+
evalType: {
|
|
223
|
+
type: 'string',
|
|
224
|
+
description: 'Heuristic-based evaluation type.',
|
|
225
|
+
enum: ['heuristic'],
|
|
226
|
+
},
|
|
227
|
+
outputType: {
|
|
228
|
+
type: 'string',
|
|
229
|
+
description: 'Float output type (0-1).',
|
|
230
|
+
enum: ['float'],
|
|
231
|
+
},
|
|
232
|
+
description: {
|
|
233
|
+
type: 'string',
|
|
234
|
+
description: 'The description of the Metric.',
|
|
235
|
+
},
|
|
236
|
+
guidelines: {
|
|
237
|
+
type: 'string',
|
|
238
|
+
description: 'Guidelines for heuristic evaluation logic.',
|
|
239
|
+
},
|
|
240
|
+
name: {
|
|
241
|
+
type: 'string',
|
|
242
|
+
description: 'The name of the Metric.',
|
|
243
|
+
},
|
|
244
|
+
passingThreshold: {
|
|
245
|
+
type: 'number',
|
|
246
|
+
description: 'Threshold for determining pass/fail from float scores (0.0-1.0).',
|
|
247
|
+
},
|
|
248
|
+
},
|
|
249
|
+
required: ['metricId', 'evalType', 'outputType'],
|
|
250
|
+
},
|
|
134
251
|
{
|
|
135
252
|
type: 'object',
|
|
136
253
|
properties: {
|
|
@@ -227,7 +344,7 @@ export const tool = {
|
|
|
227
344
|
},
|
|
228
345
|
guidelines: {
|
|
229
346
|
type: 'string',
|
|
230
|
-
description: '
|
|
347
|
+
description: 'Guidelines for heuristic evaluation logic.',
|
|
231
348
|
},
|
|
232
349
|
name: {
|
|
233
350
|
type: 'string',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"update-metrics.mjs","sourceRoot":"","sources":["../../src/tools/metrics/update-metrics.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAY,mBAAmB,EAAE,MAAM,8BAA8B;AAK5E,MAAM,CAAC,MAAM,QAAQ,GAAa;IAChC,QAAQ,EAAE,SAAS;IACnB,SAAS,EAAE,OAAO;IAClB,IAAI,EAAE,EAAE;IACR,UAAU,EAAE,OAAO;IACnB,QAAQ,EAAE,qBAAqB;IAC/B,WAAW,EAAE,cAAc;CAC5B,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAAS;IACxB,IAAI,EAAE,gBAAgB;IACtB,WAAW,EACT,wKAAwK;IAC1K,WAAW,EAAE;QACX,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE;YACL;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,2BAA2B;wBACxC,IAAI,EAAE,CAAC,IAAI,CAAC;qBACb;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,aAAa,EAAE;wBACb,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,qCAAqC;qBACnD;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0DAA0D;qBACxE;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,oEAAoE;qBAClF;oBACD,cAAc,EAAE;wBACd,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,kGAAkG;qBACrG;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0CAA0C;qBACxD;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,8BAA8B;wBAC3C,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;qBAChD;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,oEAAoE;qBAClF;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;wBAC/C,IAAI,EAAE,CAAC,WAAW,CAAC;qBACpB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,
|
|
1
|
+
{"version":3,"file":"update-metrics.mjs","sourceRoot":"","sources":["../../src/tools/metrics/update-metrics.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAY,mBAAmB,EAAE,MAAM,8BAA8B;AAK5E,MAAM,CAAC,MAAM,QAAQ,GAAa;IAChC,QAAQ,EAAE,SAAS;IACnB,SAAS,EAAE,OAAO;IAClB,IAAI,EAAE,EAAE;IACR,UAAU,EAAE,OAAO;IACnB,QAAQ,EAAE,qBAAqB;IAC/B,WAAW,EAAE,cAAc;CAC5B,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAAS;IACxB,IAAI,EAAE,gBAAgB;IACtB,WAAW,EACT,wKAAwK;IAC1K,WAAW,EAAE;QACX,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE;YACL;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,2BAA2B;wBACxC,IAAI,EAAE,CAAC,IAAI,CAAC;qBACb;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,aAAa,EAAE;wBACb,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,qCAAqC;qBACnD;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0DAA0D;qBACxE;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,oEAAoE;qBAClF;oBACD,cAAc,EAAE;wBACd,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,kGAAkG;qBACrG;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0CAA0C;qBACxD;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,8BAA8B;wBAC3C,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;qBAChD;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,oEAAoE;qBAClF;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;wBAC/C,IAAI,EAAE,CAAC,WAAW,CAAC;qBACpB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,4CAA4C;qBAC1D;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,oEAAoE;qBAClF;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,2BAA2B;wBACxC,IAAI,EAAE,CAAC,IAAI,CAAC;qBACb;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0BAA0B;wBACvC,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,aAAa,EAAE;wBACb,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,qCAAqC;qBACnD;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0DAA0D;qBACxE;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kEAAkE;qBAChF;oBACD,cAAc,EAAE;wBACd,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,kGAAkG;qBACrG;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0CAA0C;qBACxD;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,8BAA8B;wBAC3C,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0BAA0B;wBACvC,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;qBAChD;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kEAAkE;qBAChF;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;wBAC/C,IAAI,EAAE,CAAC,WAAW,CAAC;qBACpB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0BAA0B;wBACvC,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,4CAA4C;qBAC1D;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kEAAkE;qBAChF;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,2BAA2B;wBACxC,IAAI,EAAE,CAAC,IAAI,CAAC;qBACb;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,SAAS,CAAC;qBAClB;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,aAAa,EAAE;wBACb,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,qCAAqC;qBACnD;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0DAA0D;qBACxE;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,cAAc,EAAE;wBACd,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,kGAAkG;qBACrG;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0CAA0C;qBACxD;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,8BAA8B;wBAC3C,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,SAAS,CAAC;qBAClB;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;qBAChD;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;wBAC/C,IAAI,EAAE,CAAC,WAAW,CAAC;qBACpB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,SAAS,CAAC;qBAClB;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,4CAA4C;qBAC1D;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;iBACF;gBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC;aACjD;SACF;KACF;IACD,WAAW,EAAE,EAAE;CAChB,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAAE,MAAiB,EAAE,IAAyC,EAAE,EAAE;IAC5F,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,IAAW,CAAC;IAC1C,OAAO,mBAAmB,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;AAC1E,CAAC,CAAC;AAEF,eAAe,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { Metadata } from 'scorecard-ai-mcp/tools/types';
|
|
2
|
+
import { Tool } from '@modelcontextprotocol/sdk/types.js';
|
|
3
|
+
import Scorecard from 'scorecard-ai';
|
|
4
|
+
export declare const metadata: Metadata;
|
|
5
|
+
export declare const tool: Tool;
|
|
6
|
+
export declare const handler: (client: Scorecard, args: Record<string, unknown> | undefined) => Promise<import("scorecard-ai-mcp/tools/types").ToolCallResult>;
|
|
7
|
+
declare const _default: {
|
|
8
|
+
metadata: Metadata;
|
|
9
|
+
tool: {
|
|
10
|
+
[x: string]: unknown;
|
|
11
|
+
name: string;
|
|
12
|
+
inputSchema: {
|
|
13
|
+
[x: string]: unknown;
|
|
14
|
+
type: "object";
|
|
15
|
+
properties?: {
|
|
16
|
+
[x: string]: unknown;
|
|
17
|
+
} | undefined;
|
|
18
|
+
required?: string[] | undefined;
|
|
19
|
+
};
|
|
20
|
+
title?: string | undefined;
|
|
21
|
+
description?: string | undefined;
|
|
22
|
+
outputSchema?: {
|
|
23
|
+
[x: string]: unknown;
|
|
24
|
+
type: "object";
|
|
25
|
+
properties?: {
|
|
26
|
+
[x: string]: unknown;
|
|
27
|
+
} | undefined;
|
|
28
|
+
required?: string[] | undefined;
|
|
29
|
+
} | undefined;
|
|
30
|
+
annotations?: {
|
|
31
|
+
[x: string]: unknown;
|
|
32
|
+
title?: string | undefined;
|
|
33
|
+
readOnlyHint?: boolean | undefined;
|
|
34
|
+
destructiveHint?: boolean | undefined;
|
|
35
|
+
idempotentHint?: boolean | undefined;
|
|
36
|
+
openWorldHint?: boolean | undefined;
|
|
37
|
+
} | undefined;
|
|
38
|
+
_meta?: {
|
|
39
|
+
[x: string]: unknown;
|
|
40
|
+
} | undefined;
|
|
41
|
+
};
|
|
42
|
+
handler: (client: Scorecard, args: Record<string, unknown> | undefined) => Promise<import("scorecard-ai-mcp/tools/types").ToolCallResult>;
|
|
43
|
+
};
|
|
44
|
+
export default _default;
|
|
45
|
+
//# sourceMappingURL=list-records.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"list-records.d.mts","sourceRoot":"","sources":["../../src/tools/records/list-records.ts"],"names":[],"mappings":"OAGO,EAAE,QAAQ,EAAuB,MAAM,8BAA8B;OAErE,EAAE,IAAI,EAAE,MAAM,oCAAoC;OAClD,SAAS,MAAM,cAAc;AAEpC,eAAO,MAAM,QAAQ,EAAE,QAOtB,CAAC;AAEF,eAAO,MAAM,IAAI,EAAE,IAgClB,CAAC;AAEF,eAAO,MAAM,OAAO,GAAU,QAAQ,SAAS,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,mEAIzF,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;sBAJoC,SAAS,QAAQ,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;;AAM1F,wBAA2C"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { Metadata } from 'scorecard-ai-mcp/tools/types';
|
|
2
|
+
import { Tool } from '@modelcontextprotocol/sdk/types.js';
|
|
3
|
+
import Scorecard from 'scorecard-ai';
|
|
4
|
+
export declare const metadata: Metadata;
|
|
5
|
+
export declare const tool: Tool;
|
|
6
|
+
export declare const handler: (client: Scorecard, args: Record<string, unknown> | undefined) => Promise<import("scorecard-ai-mcp/tools/types").ToolCallResult>;
|
|
7
|
+
declare const _default: {
|
|
8
|
+
metadata: Metadata;
|
|
9
|
+
tool: {
|
|
10
|
+
[x: string]: unknown;
|
|
11
|
+
name: string;
|
|
12
|
+
inputSchema: {
|
|
13
|
+
[x: string]: unknown;
|
|
14
|
+
type: "object";
|
|
15
|
+
properties?: {
|
|
16
|
+
[x: string]: unknown;
|
|
17
|
+
} | undefined;
|
|
18
|
+
required?: string[] | undefined;
|
|
19
|
+
};
|
|
20
|
+
title?: string | undefined;
|
|
21
|
+
description?: string | undefined;
|
|
22
|
+
outputSchema?: {
|
|
23
|
+
[x: string]: unknown;
|
|
24
|
+
type: "object";
|
|
25
|
+
properties?: {
|
|
26
|
+
[x: string]: unknown;
|
|
27
|
+
} | undefined;
|
|
28
|
+
required?: string[] | undefined;
|
|
29
|
+
} | undefined;
|
|
30
|
+
annotations?: {
|
|
31
|
+
[x: string]: unknown;
|
|
32
|
+
title?: string | undefined;
|
|
33
|
+
readOnlyHint?: boolean | undefined;
|
|
34
|
+
destructiveHint?: boolean | undefined;
|
|
35
|
+
idempotentHint?: boolean | undefined;
|
|
36
|
+
openWorldHint?: boolean | undefined;
|
|
37
|
+
} | undefined;
|
|
38
|
+
_meta?: {
|
|
39
|
+
[x: string]: unknown;
|
|
40
|
+
} | undefined;
|
|
41
|
+
};
|
|
42
|
+
handler: (client: Scorecard, args: Record<string, unknown> | undefined) => Promise<import("scorecard-ai-mcp/tools/types").ToolCallResult>;
|
|
43
|
+
};
|
|
44
|
+
export default _default;
|
|
45
|
+
//# sourceMappingURL=list-records.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"list-records.d.ts","sourceRoot":"","sources":["../../src/tools/records/list-records.ts"],"names":[],"mappings":"OAGO,EAAE,QAAQ,EAAuB,MAAM,8BAA8B;OAErE,EAAE,IAAI,EAAE,MAAM,oCAAoC;OAClD,SAAS,MAAM,cAAc;AAEpC,eAAO,MAAM,QAAQ,EAAE,QAOtB,CAAC;AAEF,eAAO,MAAM,IAAI,EAAE,IAgClB,CAAC;AAEF,eAAO,MAAM,OAAO,GAAU,QAAQ,SAAS,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,mEAIzF,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;sBAJoC,SAAS,QAAQ,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;;AAM1F,wBAA2C"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.handler = exports.tool = exports.metadata = void 0;
|
|
5
|
+
const filtering_1 = require("scorecard-ai-mcp/filtering");
|
|
6
|
+
const types_1 = require("scorecard-ai-mcp/tools/types");
|
|
7
|
+
exports.metadata = {
|
|
8
|
+
resource: 'records',
|
|
9
|
+
operation: 'read',
|
|
10
|
+
tags: [],
|
|
11
|
+
httpMethod: 'get',
|
|
12
|
+
httpPath: '/runs/{runId}/records',
|
|
13
|
+
operationId: 'listRecords',
|
|
14
|
+
};
|
|
15
|
+
exports.tool = {
|
|
16
|
+
name: 'list_records',
|
|
17
|
+
description: "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nRetrieve a paginated list of Records for a Run, including all scores for each record.\n\n# Response Schema\n```json\n{\n type: 'object',\n properties: {\n data: {\n type: 'array',\n items: {\n $ref: '#/$defs/record_list_response'\n }\n },\n hasMore: {\n type: 'boolean'\n },\n nextCursor: {\n type: 'string'\n },\n total: {\n type: 'integer'\n }\n },\n required: [ 'data',\n 'hasMore',\n 'nextCursor'\n ],\n $defs: {\n record_list_response: {\n allOf: [ {\n $ref: '#/$defs/record'\n }\n ],\n description: 'A record with all its associated scores.'\n },\n record: {\n type: 'object',\n description: 'A record of a system execution in the Scorecard system.',\n properties: {\n id: {\n type: 'string',\n description: 'The ID of the Record.'\n },\n expected: {\n type: 'object',\n description: 'The expected outputs for the Testcase.',\n additionalProperties: true\n },\n inputs: {\n type: 'object',\n description: 'The actual inputs sent to the system, which should match the system\\'s input schema.',\n additionalProperties: true\n },\n outputs: {\n type: 'object',\n description: 'The actual outputs from the system.',\n additionalProperties: true\n },\n runId: {\n type: 'string',\n description: 'The ID of the Run containing this Record.'\n },\n testcaseId: {\n type: 'string',\n description: 'The ID of the Testcase.'\n }\n },\n required: [ 'id',\n 'expected',\n 'inputs',\n 'outputs',\n 'runId'\n ]\n }\n }\n}\n```",
|
|
18
|
+
inputSchema: {
|
|
19
|
+
type: 'object',
|
|
20
|
+
properties: {
|
|
21
|
+
runId: {
|
|
22
|
+
type: 'string',
|
|
23
|
+
},
|
|
24
|
+
cursor: {
|
|
25
|
+
type: 'string',
|
|
26
|
+
description: 'Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of results.',
|
|
27
|
+
},
|
|
28
|
+
limit: {
|
|
29
|
+
type: 'integer',
|
|
30
|
+
description: 'Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.',
|
|
31
|
+
},
|
|
32
|
+
jq_filter: {
|
|
33
|
+
type: 'string',
|
|
34
|
+
title: 'jq Filter',
|
|
35
|
+
description: 'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
|
|
36
|
+
},
|
|
37
|
+
},
|
|
38
|
+
required: ['runId'],
|
|
39
|
+
},
|
|
40
|
+
annotations: {
|
|
41
|
+
readOnlyHint: true,
|
|
42
|
+
},
|
|
43
|
+
};
|
|
44
|
+
const handler = async (client, args) => {
|
|
45
|
+
const { runId, jq_filter, ...body } = args;
|
|
46
|
+
const response = await client.records.list(runId, body).asResponse();
|
|
47
|
+
return (0, types_1.asTextContentResult)(await (0, filtering_1.maybeFilter)(jq_filter, await response.json()));
|
|
48
|
+
};
|
|
49
|
+
exports.handler = handler;
|
|
50
|
+
exports.default = { metadata: exports.metadata, tool: exports.tool, handler: exports.handler };
|
|
51
|
+
//# sourceMappingURL=list-records.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"list-records.js","sourceRoot":"","sources":["../../src/tools/records/list-records.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,0DAAyD;AACzD,wDAA6E;AAKhE,QAAA,QAAQ,GAAa;IAChC,QAAQ,EAAE,SAAS;IACnB,SAAS,EAAE,MAAM;IACjB,IAAI,EAAE,EAAE;IACR,UAAU,EAAE,KAAK;IACjB,QAAQ,EAAE,uBAAuB;IACjC,WAAW,EAAE,aAAa;CAC3B,CAAC;AAEW,QAAA,IAAI,GAAS;IACxB,IAAI,EAAE,cAAc;IACpB,WAAW,EACT,+8DAA+8D;IACj9D,WAAW,EAAE;QACX,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;aACf;YACD,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,WAAW,EACT,0GAA0G;aAC7G;YACD,KAAK,EAAE;gBACL,IAAI,EAAE,SAAS;gBACf,WAAW,EACT,iGAAiG;aACpG;YACD,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,WAAW;gBAClB,WAAW,EACT,kWAAkW;aACrW;SACF;QACD,QAAQ,EAAE,CAAC,OAAO,CAAC;KACpB;IACD,WAAW,EAAE;QACX,YAAY,EAAE,IAAI;KACnB;CACF,CAAC;AAEK,MAAM,OAAO,GAAG,KAAK,EAAE,MAAiB,EAAE,IAAyC,EAAE,EAAE;IAC5F,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,GAAG,IAAW,CAAC;IAClD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;IACrE,OAAO,IAAA,2BAAmB,EAAC,MAAM,IAAA,uBAAW,EAAC,SAAS,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;AAClF,CAAC,CAAC;AAJW,QAAA,OAAO,WAIlB;AAEF,kBAAe,EAAE,QAAQ,EAAR,gBAAQ,EAAE,IAAI,EAAJ,YAAI,EAAE,OAAO,EAAP,eAAO,EAAE,CAAC"}
|