scorecard-ai-mcp 1.0.0-alpha.9 → 1.1.0

package/tools/metrics/create-metrics.mjs

@@ -0,0 +1,241 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+export const metadata = {
+    resource: 'metrics',
+    operation: 'write',
+    tags: [],
+    httpMethod: 'post',
+    httpPath: '/projects/{projectId}/metrics',
+    operationId: 'createMetric',
+};
+export const tool = {
+    name: 'create_metrics',
+    description: 'Create a new Metric for evaluating system outputs. The structure of a metric depends on the evalType and outputType of the metric.',
+    inputSchema: {
+        type: 'object',
+        anyOf: [
+            {
+                type: 'object',
+                properties: {
+                    projectId: {
+                        type: 'string',
+                    },
+                    evalType: {
+                        type: 'string',
+                        description: 'AI-based evaluation type.',
+                        enum: ['ai'],
+                    },
+                    name: {
+                        type: 'string',
+                        description: 'The name of the Metric.',
+                    },
+                    outputType: {
+                        type: 'string',
+                        description: 'Integer output type.',
+                        enum: ['int'],
+                    },
+                    promptTemplate: {
+                        type: 'string',
+                        description: 'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
+                    },
+                    description: {
+                        type: 'string',
+                        description: 'The description of the Metric.',
+                    },
+                    evalModelName: {
+                        type: 'string',
+                        description: 'The AI model to use for evaluation.',
+                    },
+                    guidelines: {
+                        type: 'string',
+                        description: 'Guidelines for AI evaluation on how to score the metric.',
+                    },
+                    passingThreshold: {
+                        type: 'integer',
+                        description: 'The threshold for determining pass/fail from integer scores (1-5).',
+                    },
+                    temperature: {
+                        type: 'number',
+                        description: 'The temperature for AI evaluation (0-2).',
+                    },
+                },
+            },
+            {
+                type: 'object',
+                properties: {
+                    projectId: {
+                        type: 'string',
+                    },
+                    evalType: {
+                        type: 'string',
+                        description: 'Human-based evaluation type.',
+                        enum: ['human'],
+                    },
+                    name: {
+                        type: 'string',
+                        description: 'The name of the Metric.',
+                    },
+                    outputType: {
+                        type: 'string',
+                        description: 'Integer output type.',
+                        enum: ['int'],
+                    },
+                    description: {
+                        type: 'string',
+                        description: 'The description of the Metric.',
+                    },
+                    guidelines: {
+                        type: 'string',
+                        description: 'Guidelines for human evaluators.',
+                    },
+                    passingThreshold: {
+                        type: 'integer',
+                        description: 'The threshold for determining pass/fail from integer scores (1-5).',
+                    },
+                },
+            },
+            {
+                type: 'object',
+                properties: {
+                    projectId: {
+                        type: 'string',
+                    },
+                    evalType: {
+                        type: 'string',
+                        description: 'Heuristic-based evaluation type.',
+                        enum: ['heuristic'],
+                    },
+                    name: {
+                        type: 'string',
+                        description: 'The name of the Metric.',
+                    },
+                    outputType: {
+                        type: 'string',
+                        description: 'Integer output type.',
+                        enum: ['int'],
+                    },
+                    description: {
+                        type: 'string',
+                        description: 'The description of the Metric.',
+                    },
+                    guidelines: {
+                        type: 'string',
+                        description: 'Optional guidelines for heuristic evaluation logic.',
+                    },
+                    passingThreshold: {
+                        type: 'integer',
+                        description: 'The threshold for determining pass/fail from integer scores (1-5).',
+                    },
+                },
+            },
+            {
+                type: 'object',
+                properties: {
+                    projectId: {
+                        type: 'string',
+                    },
+                    evalType: {
+                        type: 'string',
+                        description: 'AI-based evaluation type.',
+                        enum: ['ai'],
+                    },
+                    name: {
+                        type: 'string',
+                        description: 'The name of the Metric.',
+                    },
+                    outputType: {
+                        type: 'string',
+                        description: 'Boolean output type.',
+                        enum: ['boolean'],
+                    },
+                    promptTemplate: {
+                        type: 'string',
+                        description: 'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
+                    },
+                    description: {
+                        type: 'string',
+                        description: 'The description of the Metric.',
+                    },
+                    evalModelName: {
+                        type: 'string',
+                        description: 'The AI model to use for evaluation.',
+                    },
+                    guidelines: {
+                        type: 'string',
+                        description: 'Guidelines for AI evaluation on how to score the metric.',
+                    },
+                    temperature: {
+                        type: 'number',
+                        description: 'The temperature for AI evaluation (0-2).',
+                    },
+                },
+            },
+            {
+                type: 'object',
+                properties: {
+                    projectId: {
+                        type: 'string',
+                    },
+                    evalType: {
+                        type: 'string',
+                        description: 'Human-based evaluation type.',
+                        enum: ['human'],
+                    },
+                    name: {
+                        type: 'string',
+                        description: 'The name of the Metric.',
+                    },
+                    outputType: {
+                        type: 'string',
+                        description: 'Boolean output type.',
+                        enum: ['boolean'],
+                    },
+                    description: {
+                        type: 'string',
+                        description: 'The description of the Metric.',
+                    },
+                    guidelines: {
+                        type: 'string',
+                        description: 'Guidelines for human evaluators.',
+                    },
+                },
+            },
+            {
+                type: 'object',
+                properties: {
+                    projectId: {
+                        type: 'string',
+                    },
+                    evalType: {
+                        type: 'string',
+                        description: 'Heuristic-based evaluation type.',
+                        enum: ['heuristic'],
+                    },
+                    name: {
+                        type: 'string',
+                        description: 'The name of the Metric.',
+                    },
+                    outputType: {
+                        type: 'string',
+                        description: 'Boolean output type.',
+                        enum: ['boolean'],
+                    },
+                    description: {
+                        type: 'string',
+                        description: 'The description of the Metric.',
+                    },
+                    guidelines: {
+                        type: 'string',
+                        description: 'Optional guidelines for heuristic evaluation logic.',
+                    },
+                },
+            },
+        ],
+    },
+};
+export const handler = async (client, args) => {
+    const { projectId, ...body } = args;
+    return asTextContentResult(await client.metrics.create(projectId, body));
+};
+export default { metadata, tool, handler };
+//# sourceMappingURL=create-metrics.mjs.map

package/tools/metrics/create-metrics.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"create-metrics.mjs","sourceRoot":"","sources":["../../src/tools/metrics/create-metrics.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,mBAAmB,EAAE,MAAM,8BAA8B;AAMlE,MAAM,CAAC,MAAM,QAAQ,GAAa;IAChC,QAAQ,EAAE,SAAS;IACnB,SAAS,EAAE,OAAO;IAClB,IAAI,EAAE,EAAE;IACR,UAAU,EAAE,MAAM;IAClB,QAAQ,EAAE,+BAA+B;IACzC,WAAW,EAAE,cAAc;CAC5B,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAAS;IACxB,IAAI,EAAE,gBAAgB;IACtB,WAAW,EACT,oIAAoI;IACtI,WAAW,EAAE;QACX,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE;YACL;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,SAAS,EAAE;wBACT,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,2BAA2B;wBACxC,IAAI,EAAE,CAAC,IAAI,CAAC;qBACb;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,cAAc,EAAE;wBACd,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,kGAAkG;qBACrG;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,aAAa,EAAE;wBACb,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,qCAAqC;qBACnD;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0DAA0D;qBACxE;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,oEAAoE;qBAClF;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0CAA0C;qBACxD;iBACF;aACF;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,SAAS,EAAE;wBACT,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,8BAA8B;wBAC3C,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;qBAChD;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,oEAAoE;qBAClF;iBACF;aACF;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,SAAS,EAAE;wBACT,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;wBAC/C,IAAI,EAAE,CAAC,WAAW,CAAC;qBACpB;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,qDAAqD;qBACnE;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,oEAAoE;qBAClF;iBACF;aACF;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,SAAS,EAAE;wBACT,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,2BAA2B;wBACxC,IAAI,EAAE,CAAC,IAAI,CAAC;qBACb;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,SAAS,CAAC;qBAClB;oBACD,cAAc,EAAE;wBACd,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,kGAAkG;qBACrG;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,aAAa,EAAE;wBACb,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,qCAAqC;qBACnD;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0DAA0D;qBACxE;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0CAA0C;qBACxD;iBACF;aACF;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,SAAS,EAAE;wBACT,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,8BAA8B;wBAC3C,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,SAAS,CAAC;qBAClB;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;qBAChD;iBACF;aACF;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,SAAS,EAAE;wBACT,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;wBAC/C,IAAI,EAAE,CAAC,WAAW,CAAC;qBACpB;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,SAAS,CAAC;qBAClB;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,qDAAqD;qBACnE;iBACF;aACF;SACF;KACF;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAAE,MAAiB,EAAE,IAAyC,EAAE,EAAE;IAC5F,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,GAAG,IAAW,CAAC;IAC3C,OAAO,mBAAmB,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;AAC3E,CAAC,CAAC;AAEF,eAAe,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC"}

package/tools/{system-configs/get-system-configs.d.mts → metrics/update-metrics.d.mts}

@@ -15,8 +15,17 @@ declare const _default: {
             properties?: {
                 [x: string]: unknown;
             } | undefined;
+            required?: string[] | undefined;
         };
         description?: string | undefined;
+        outputSchema?: {
+            [x: string]: unknown;
+            type: "object";
+            properties?: {
+                [x: string]: unknown;
+            } | undefined;
+            required?: string[] | undefined;
+        } | undefined;
         annotations?: {
             [x: string]: unknown;
             title?: string | undefined;
@@ -29,4 +38,4 @@ declare const _default: {
     handler: (client: Scorecard, args: Record<string, unknown> | undefined) => Promise<import("scorecard-ai-mcp/tools/types").ToolCallResult>;
 };
 export default _default;
-//# sourceMappingURL=get-system-configs.d.mts.map
+//# sourceMappingURL=update-metrics.d.mts.map

package/tools/metrics/update-metrics.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"update-metrics.d.mts","sourceRoot":"","sources":["../../src/tools/metrics/update-metrics.ts"],"names":[],"mappings":"OAIO,EAAE,IAAI,EAAE,MAAM,oCAAoC;OAClD,KAAK,EAAE,QAAQ,EAAE;OACjB,SAAS,MAAM,cAAc;AAEpC,eAAO,MAAM,QAAQ,EAAE,QAOtB,CAAC;AAEF,eAAO,MAAM,IAAI,EAAE,IAmOlB,CAAC;AAEF,eAAO,MAAM,OAAO,GAAU,QAAQ,SAAS,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,mEAGzF,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;sBAHoC,SAAS,QAAQ,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;;AAK1F,wBAA2C"}

package/tools/{system-configs/get-system-configs.d.ts → metrics/update-metrics.d.ts}

@@ -15,8 +15,17 @@ declare const _default: {
             properties?: {
                 [x: string]: unknown;
             } | undefined;
+            required?: string[] | undefined;
         };
         description?: string | undefined;
+        outputSchema?: {
+            [x: string]: unknown;
+            type: "object";
+            properties?: {
+                [x: string]: unknown;
+            } | undefined;
+            required?: string[] | undefined;
+        } | undefined;
         annotations?: {
             [x: string]: unknown;
             title?: string | undefined;
@@ -29,4 +38,4 @@ declare const _default: {
     handler: (client: Scorecard, args: Record<string, unknown> | undefined) => Promise<import("scorecard-ai-mcp/tools/types").ToolCallResult>;
 };
 export default _default;
-//# sourceMappingURL=get-system-configs.d.ts.map
+//# sourceMappingURL=update-metrics.d.ts.map

package/tools/metrics/update-metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"update-metrics.d.ts","sourceRoot":"","sources":["../../src/tools/metrics/update-metrics.ts"],"names":[],"mappings":"OAIO,EAAE,IAAI,EAAE,MAAM,oCAAoC;OAClD,KAAK,EAAE,QAAQ,EAAE;OACjB,SAAS,MAAM,cAAc;AAEpC,eAAO,MAAM,QAAQ,EAAE,QAOtB,CAAC;AAEF,eAAO,MAAM,IAAI,EAAE,IAmOlB,CAAC;AAEF,eAAO,MAAM,OAAO,GAAU,QAAQ,SAAS,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,mEAGzF,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;sBAHoC,SAAS,QAAQ,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;;AAK1F,wBAA2C"}

package/tools/metrics/update-metrics.js

@@ -0,0 +1,245 @@
+"use strict";
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handler = exports.tool = exports.metadata = void 0;
+const types_1 = require("scorecard-ai-mcp/tools/types");
+exports.metadata = {
+    resource: 'metrics',
+    operation: 'write',
+    tags: [],
+    httpMethod: 'patch',
+    httpPath: '/metrics/{metricId}',
+    operationId: 'updateMetric',
+};
+exports.tool = {
+    name: 'update_metrics',
+    description: 'Update an existing Metric. You must specify the evalType and outputType of the metric. The structure of a metric depends on the evalType and outputType of the metric.',
+    inputSchema: {
+        type: 'object',
+        anyOf: [
+            {
+                type: 'object',
+                properties: {
+                    metricId: {
+                        type: 'string',
+                    },
+                    evalType: {
+                        type: 'string',
+                        description: 'AI-based evaluation type.',
+                        enum: ['ai'],
+                    },
+                    outputType: {
+                        type: 'string',
+                        description: 'Integer output type.',
+                        enum: ['int'],
+                    },
+                    description: {
+                        type: 'string',
+                        description: 'The description of the Metric.',
+                    },
+                    evalModelName: {
+                        type: 'string',
+                        description: 'The AI model to use for evaluation.',
+                    },
+                    guidelines: {
+                        type: 'string',
+                        description: 'Guidelines for AI evaluation on how to score the metric.',
+                    },
+                    name: {
+                        type: 'string',
+                        description: 'The name of the Metric.',
+                    },
+                    passingThreshold: {
+                        type: 'integer',
+                        description: 'The threshold for determining pass/fail from integer scores (1-5).',
+                    },
+                    promptTemplate: {
+                        type: 'string',
+                        description: 'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
+                    },
+                    temperature: {
+                        type: 'number',
+                        description: 'The temperature for AI evaluation (0-2).',
+                    },
+                },
+            },
+            {
+                type: 'object',
+                properties: {
+                    metricId: {
+                        type: 'string',
+                    },
+                    evalType: {
+                        type: 'string',
+                        description: 'Human-based evaluation type.',
+                        enum: ['human'],
+                    },
+                    outputType: {
+                        type: 'string',
+                        description: 'Integer output type.',
+                        enum: ['int'],
+                    },
+                    description: {
+                        type: 'string',
+                        description: 'The description of the Metric.',
+                    },
+                    guidelines: {
+                        type: 'string',
+                        description: 'Guidelines for human evaluators.',
+                    },
+                    name: {
+                        type: 'string',
+                        description: 'The name of the Metric.',
+                    },
+                    passingThreshold: {
+                        type: 'integer',
+                        description: 'The threshold for determining pass/fail from integer scores (1-5).',
+                    },
+                },
+            },
+            {
+                type: 'object',
+                properties: {
+                    metricId: {
+                        type: 'string',
+                    },
+                    evalType: {
+                        type: 'string',
+                        description: 'Heuristic-based evaluation type.',
+                        enum: ['heuristic'],
+                    },
+                    outputType: {
+                        type: 'string',
+                        description: 'Integer output type.',
+                        enum: ['int'],
+                    },
+                    description: {
+                        type: 'string',
+                        description: 'The description of the Metric.',
+                    },
+                    guidelines: {
+                        type: 'string',
+                        description: 'Optional guidelines for heuristic evaluation logic.',
+                    },
+                    name: {
+                        type: 'string',
+                        description: 'The name of the Metric.',
+                    },
+                    passingThreshold: {
+                        type: 'integer',
+                        description: 'The threshold for determining pass/fail from integer scores (1-5).',
+                    },
+                },
+            },
+            {
+                type: 'object',
+                properties: {
+                    metricId: {
+                        type: 'string',
+                    },
+                    evalType: {
+                        type: 'string',
+                        description: 'AI-based evaluation type.',
+                        enum: ['ai'],
+                    },
+                    outputType: {
+                        type: 'string',
+                        description: 'Boolean output type.',
+                        enum: ['boolean'],
+                    },
+                    description: {
+                        type: 'string',
+                        description: 'The description of the Metric.',
+                    },
+                    evalModelName: {
+                        type: 'string',
+                        description: 'The AI model to use for evaluation.',
+                    },
+                    guidelines: {
+                        type: 'string',
+                        description: 'Guidelines for AI evaluation on how to score the metric.',
+                    },
+                    name: {
+                        type: 'string',
+                        description: 'The name of the Metric.',
+                    },
+                    promptTemplate: {
+                        type: 'string',
+                        description: 'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
+                    },
+                    temperature: {
+                        type: 'number',
+                        description: 'The temperature for AI evaluation (0-2).',
+                    },
+                },
+            },
+            {
+                type: 'object',
+                properties: {
+                    metricId: {
+                        type: 'string',
+                    },
+                    evalType: {
+                        type: 'string',
+                        description: 'Human-based evaluation type.',
+                        enum: ['human'],
+                    },
+                    outputType: {
+                        type: 'string',
+                        description: 'Boolean output type.',
+                        enum: ['boolean'],
+                    },
+                    description: {
+                        type: 'string',
+                        description: 'The description of the Metric.',
+                    },
+                    guidelines: {
+                        type: 'string',
+                        description: 'Guidelines for human evaluators.',
+                    },
+                    name: {
+                        type: 'string',
+                        description: 'The name of the Metric.',
+                    },
+                },
+            },
+            {
+                type: 'object',
+                properties: {
+                    metricId: {
+                        type: 'string',
+                    },
+                    evalType: {
+                        type: 'string',
+                        description: 'Heuristic-based evaluation type.',
+                        enum: ['heuristic'],
+                    },
+                    outputType: {
+                        type: 'string',
+                        description: 'Boolean output type.',
+                        enum: ['boolean'],
+                    },
+                    description: {
+                        type: 'string',
+                        description: 'The description of the Metric.',
+                    },
+                    guidelines: {
+                        type: 'string',
+                        description: 'Optional guidelines for heuristic evaluation logic.',
+                    },
+                    name: {
+                        type: 'string',
+                        description: 'The name of the Metric.',
+                    },
+                },
+            },
+        ],
+    },
+};
+const handler = async (client, args) => {
+    const { metricId, ...body } = args;
+    return (0, types_1.asTextContentResult)(await client.metrics.update(metricId, body));
+};
+exports.handler = handler;
+exports.default = { metadata: exports.metadata, tool: exports.tool, handler: exports.handler };
+//# sourceMappingURL=update-metrics.js.map

package/tools/metrics/update-metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"update-metrics.js","sourceRoot":"","sources":["../../src/tools/metrics/update-metrics.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,wDAAmE;AAMtD,QAAA,QAAQ,GAAa;IAChC,QAAQ,EAAE,SAAS;IACnB,SAAS,EAAE,OAAO;IAClB,IAAI,EAAE,EAAE;IACR,UAAU,EAAE,OAAO;IACnB,QAAQ,EAAE,qBAAqB;IAC/B,WAAW,EAAE,cAAc;CAC5B,CAAC;AAEW,QAAA,IAAI,GAAS;IACxB,IAAI,EAAE,gBAAgB;IACtB,WAAW,EACT,wKAAwK;IAC1K,WAAW,EAAE;QACX,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE;YACL;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,2BAA2B;wBACxC,IAAI,EAAE,CAAC,IAAI,CAAC;qBACb;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,aAAa,EAAE;wBACb,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,qCAAqC;qBACnD;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0DAA0D;qBACxE;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,oEAAoE;qBAClF;oBACD,cAAc,EAAE;wBACd,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,kGAAkG;qBACrG;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0CAA0C;qBACxD;iBACF;aACF;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,8BAA8B;wBAC3C,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;qBAChD;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,oEAAoE;qBAClF;iBACF;aACF;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;wBAC/C,IAAI,EAAE,CAAC,WAAW,CAAC;qBACpB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,KAAK,CAAC;qBACd;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,qDAAqD;qBACnE;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,gBAAgB,EAAE;wBAChB,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,oEAAoE;qBAClF;iBACF;aACF;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,2BAA2B;wBACxC,IAAI,EAAE,CAAC,IAAI,CAAC;qBACb;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,SAAS,CAAC;qBAClB;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,aAAa,EAAE;wBACb,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,qCAAqC;qBACnD;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0DAA0D;qBACxE;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;oBACD,cAAc,EAAE;wBACd,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,kGAAkG;qBACrG;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0CAA0C;qBACxD;iBACF;aACF;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,8BAA8B;wBAC3C,IAAI,EAAE,CAAC,OAAO,CAAC;qBAChB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,SAAS,CAAC;qBAClB;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;qBAChD;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;iBACF;aACF;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;wBAC/C,IAAI,EAAE,CAAC,WAAW,CAAC;qBACpB;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,CAAC,SAAS,CAAC;qBAClB;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,qDAAqD;qBACnE;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,yBAAyB;qBACvC;iBACF;aACF;SACF;KACF;CACF,CAAC;AAEK,MAAM,OAAO,GAAG,KAAK,EAAE,MAAiB,EAAE,IAAyC,EAAE,EAAE;IAC5F,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,IAAW,CAAC;IAC1C,OAAO,IAAA,2BAAmB,EAAC,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;AAC1E,CAAC,CAAC;AAHW,QAAA,OAAO,WAGlB;AAEF,kBAAe,EAAE,QAAQ,EAAR,gBAAQ,EAAE,IAAI,EAAJ,YAAI,EAAE,OAAO,EAAP,eAAO,EAAE,CAAC"}