scorecard-ai-mcp 1.0.0-alpha.8 → 1.0.0

package/src/tools/metrics/create-metrics.ts

@@ -0,0 +1,251 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'metrics',
+  operation: 'write',
+  tags: [],
+  httpMethod: 'post',
+  httpPath: '/projects/{projectId}/metrics',
+  operationId: 'createMetric',
+};
+
+export const tool: Tool = {
+  name: 'create_metrics',
+  description: 'Create a new Metric for evaluating system outputs.',
+  inputSchema: {
+    type: 'object',
+    anyOf: [
+      {
+        type: 'object',
+        properties: {
+          projectId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'AI-based evaluation type.',
+            enum: ['ai'],
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+          outputType: {
+            type: 'string',
+            description: 'Integer output type.',
+            enum: ['int'],
+          },
+          promptTemplate: {
+            type: 'string',
+            description:
+              'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          evalModelName: {
+            type: 'string',
+            description: 'The AI model to use for evaluation.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Guidelines for AI evaluation on how to score the metric.',
+          },
+          passingThreshold: {
+            type: 'integer',
+            description: 'The threshold for determining pass/fail from integer scores (1-5).',
+          },
+          temperature: {
+            type: 'number',
+            description: 'The temperature for AI evaluation (0-2).',
+          },
+        },
+      },
+      {
+        type: 'object',
+        properties: {
+          projectId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'Human-based evaluation type.',
+            enum: ['human'],
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+          outputType: {
+            type: 'string',
+            description: 'Integer output type.',
+            enum: ['int'],
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Guidelines for human evaluators.',
+          },
+          passingThreshold: {
+            type: 'integer',
+            description: 'The threshold for determining pass/fail from integer scores (1-5).',
+          },
+        },
+      },
+      {
+        type: 'object',
+        properties: {
+          projectId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'Heuristic-based evaluation type.',
+            enum: ['heuristic'],
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+          outputType: {
+            type: 'string',
+            description: 'Integer output type.',
+            enum: ['int'],
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Optional guidelines for heuristic evaluation logic.',
+          },
+          passingThreshold: {
+            type: 'integer',
+            description: 'The threshold for determining pass/fail from integer scores (1-5).',
+          },
+        },
+      },
+      {
+        type: 'object',
+        properties: {
+          projectId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'AI-based evaluation type.',
+            enum: ['ai'],
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+          outputType: {
+            type: 'string',
+            description: 'Boolean output type.',
+            enum: ['boolean'],
+          },
+          promptTemplate: {
+            type: 'string',
+            description:
+              'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          evalModelName: {
+            type: 'string',
+            description: 'The AI model to use for evaluation.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Guidelines for AI evaluation on how to score the metric.',
+          },
+          temperature: {
+            type: 'number',
+            description: 'The temperature for AI evaluation (0-2).',
+          },
+        },
+      },
+      {
+        type: 'object',
+        properties: {
+          projectId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'Human-based evaluation type.',
+            enum: ['human'],
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+          outputType: {
+            type: 'string',
+            description: 'Boolean output type.',
+            enum: ['boolean'],
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Guidelines for human evaluators.',
+          },
+        },
+      },
+      {
+        type: 'object',
+        properties: {
+          projectId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'Heuristic-based evaluation type.',
+            enum: ['heuristic'],
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+          outputType: {
+            type: 'string',
+            description: 'Boolean output type.',
+            enum: ['boolean'],
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Optional guidelines for heuristic evaluation logic.',
+          },
+        },
+      },
+    ],
+  },
+};
+
+export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { projectId, ...body } = args as any;
+  return asTextContentResult(await client.metrics.create(projectId, body));
+};
+
+export default { metadata, tool, handler };

package/src/tools/projects/create-projects.ts

@@ -1,5 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
+import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 import type { Metadata } from '../';
 import Scorecard from 'scorecard-ai';
@@ -8,6 +10,9 @@ export const metadata: Metadata = {
   resource: 'projects',
   operation: 'write',
   tags: [],
+  httpMethod: 'post',
+  httpPath: '/projects',
+  operationId: 'createProject',
 };
 
 export const tool: Tool = {
@@ -28,9 +33,9 @@ export const tool: Tool = {
   },
 };
 
-export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
   const body = args as any;
-  return client.projects.create(body);
+  return asTextContentResult(await client.projects.create(body));
 };
 
 export default { metadata, tool, handler };

package/src/tools/projects/list-projects.ts

@@ -1,5 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
+import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 import type { Metadata } from '../';
 import Scorecard from 'scorecard-ai';
@@ -8,6 +10,9 @@ export const metadata: Metadata = {
   resource: 'projects',
   operation: 'read',
   tags: [],
+  httpMethod: 'get',
+  httpPath: '/projects',
+  operationId: 'listProjects',
 };
 
 export const tool: Tool = {
@@ -31,9 +36,9 @@ export const tool: Tool = {
   },
 };
 
-export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
   const body = args as any;
-  return client.projects.list(body);
+  return asTextContentResult(await client.projects.list(body));
 };
 
 export default { metadata, tool, handler };

package/src/tools/records/create-records.ts

@@ -1,5 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
+import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 import type { Metadata } from '../';
 import Scorecard from 'scorecard-ai';
@@ -8,6 +10,9 @@ export const metadata: Metadata = {
   resource: 'records',
   operation: 'write',
   tags: [],
+  httpMethod: 'post',
+  httpPath: '/runs/{runId}/records',
+  operationId: 'createRecord',
 };
 
 export const tool: Tool = {
@@ -39,9 +44,9 @@ export const tool: Tool = {
   },
 };
 
-export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
   const { runId, ...body } = args as any;
-  return client.records.create(runId, body);
+  return asTextContentResult(await client.records.create(runId, body));
 };
 
 export default { metadata, tool, handler };

package/src/tools/runs/create-runs.ts

@@ -1,5 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
+import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 import type { Metadata } from '../';
 import Scorecard from 'scorecard-ai';
@@ -8,6 +10,9 @@ export const metadata: Metadata = {
   resource: 'runs',
   operation: 'write',
   tags: [],
+  httpMethod: 'post',
+  httpPath: '/projects/{projectId}/runs',
+  operationId: 'createRun',
 };
 
 export const tool: Tool = {
@@ -26,21 +31,21 @@ export const tool: Tool = {
           type: 'string',
         },
       },
-      testsetId: {
+      systemVersionId: {
         type: 'string',
-        description: 'The ID of the Testset this Run is testing.',
+        description: 'The ID of the system version this Run is using.',
       },
-      systemConfigId: {
+      testsetId: {
         type: 'string',
-        description: 'The ID of the system configuration this Run is using.',
+        description: 'The ID of the Testset this Run is testing.',
       },
     },
   },
 };
 
-export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
   const { projectId, ...body } = args as any;
-  return client.runs.create(projectId, body);
+  return asTextContentResult(await client.runs.create(projectId, body));
 };
 
 export default { metadata, tool, handler };

package/src/tools/scores/upsert-scores.ts

@@ -1,5 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
+import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 import type { Metadata } from '../';
 import Scorecard from 'scorecard-ai';
@@ -8,6 +10,9 @@ export const metadata: Metadata = {
   resource: 'scores',
   operation: 'write',
   tags: [],
+  httpMethod: 'put',
+  httpPath: '/records/{recordId}/scores/{metricConfigId}',
+  operationId: 'upsertScore',
 };
 
 export const tool: Tool = {
@@ -32,9 +37,9 @@ export const tool: Tool = {
   },
 };
 
-export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
   const { metricConfigId, ...body } = args as any;
-  return client.scores.upsert(metricConfigId, body);
+  return asTextContentResult(await client.scores.upsert(metricConfigId, body));
 };
 
 export default { metadata, tool, handler };

package/src/tools/systems/create-systems.ts

@@ -1,5 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
+import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 import type { Metadata } from '../';
 import Scorecard from 'scorecard-ai';
@@ -8,6 +10,9 @@ export const metadata: Metadata = {
   resource: 'systems',
   operation: 'write',
   tags: [],
+  httpMethod: 'post',
+  httpPath: '/projects/{projectId}/systems',
+  operationId: 'createSystem',
 };
 
 export const tool: Tool = {
@@ -44,9 +49,9 @@ export const tool: Tool = {
   },
 };
 
-export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
   const { projectId, ...body } = args as any;
-  return client.systems.create(projectId, body);
+  return asTextContentResult(await client.systems.create(projectId, body));
 };
 
 export default { metadata, tool, handler };

package/src/tools/systems/delete-systems.ts

@@ -1,5 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
+import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 import type { Metadata } from '../';
 import Scorecard from 'scorecard-ai';
@@ -8,11 +10,14 @@ export const metadata: Metadata = {
   resource: 'systems',
   operation: 'write',
   tags: [],
+  httpMethod: 'delete',
+  httpPath: '/systems/{systemId}',
+  operationId: 'deleteSystem',
 };
 
 export const tool: Tool = {
   name: 'delete_systems',
-  description: 'Delete a system definition by ID. This will not delete associated system configurations.',
+  description: 'Delete a system definition by ID. This will not delete associated system versions.',
   inputSchema: {
     type: 'object',
     properties: {
@@ -23,9 +28,9 @@ export const tool: Tool = {
   },
 };
 
-export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
   const { systemId, ...body } = args as any;
-  return client.systems.delete(systemId);
+  return asTextContentResult(await client.systems.delete(systemId));
 };
 
 export default { metadata, tool, handler };

package/src/tools/systems/get-systems.ts

@@ -1,5 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
+import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 import type { Metadata } from '../';
 import Scorecard from 'scorecard-ai';
@@ -8,6 +10,9 @@ export const metadata: Metadata = {
   resource: 'systems',
   operation: 'read',
   tags: [],
+  httpMethod: 'get',
+  httpPath: '/systems/{systemId}',
+  operationId: 'getSystem',
 };
 
 export const tool: Tool = {
@@ -23,9 +28,9 @@ export const tool: Tool = {
   },
 };
 
-export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
   const { systemId, ...body } = args as any;
-  return client.systems.get(systemId);
+  return asTextContentResult(await client.systems.get(systemId));
 };
 
 export default { metadata, tool, handler };

package/src/tools/systems/list-systems.ts

@@ -1,5 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
+import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 import type { Metadata } from '../';
 import Scorecard from 'scorecard-ai';
@@ -8,6 +10,9 @@ export const metadata: Metadata = {
   resource: 'systems',
   operation: 'read',
   tags: [],
+  httpMethod: 'get',
+  httpPath: '/projects/{projectId}/systems',
+  operationId: 'listSystems',
 };
 
 export const tool: Tool = {
@@ -33,9 +38,9 @@ export const tool: Tool = {
   },
 };
 
-export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
   const { projectId, ...body } = args as any;
-  return client.systems.list(projectId, body);
+  return asTextContentResult(await client.systems.list(projectId, body));
 };
 
 export default { metadata, tool, handler };

package/src/tools/systems/update-systems.ts

@@ -1,5 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
+import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 import type { Metadata } from '../';
 import Scorecard from 'scorecard-ai';
@@ -8,6 +10,9 @@ export const metadata: Metadata = {
   resource: 'systems',
   operation: 'write',
   tags: [],
+  httpMethod: 'patch',
+  httpPath: '/systems/{systemId}',
+  operationId: 'updateSystem',
 };
 
 export const tool: Tool = {
@@ -44,9 +49,9 @@ export const tool: Tool = {
   },
 };
 
-export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
   const { systemId, ...body } = args as any;
-  return client.systems.update(systemId, body);
+  return asTextContentResult(await client.systems.update(systemId, body));
 };
 
 export default { metadata, tool, handler };

package/src/tools/systems/versions/create-systems-versions.ts

@@ -0,0 +1,45 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'systems.versions',
+  operation: 'write',
+  tags: [],
+  httpMethod: 'post',
+  httpPath: '/systems/{systemId}/configs',
+  operationId: 'createSystemVersion',
+};
+
+export const tool: Tool = {
+  name: 'create_systems_versions',
+  description:
+    "Create a new version for a system.\n\nEach version contains specific parameter values that match the system's `configSchema` - things like model parameters, thresholds, or processing options.\nOnce created, versions cannot be modified, ensuring stable reference points for evaluations.\n\nWhen creating a system version:\n- The `config` object is validated against the parent system's `configSchema`.\n- System versions with validation errors are still stored, with errors included in the response.\n- Validation errors indicate fields that don't match the schema but don't prevent creation.\n- Having validation errors may affect how some evaluation metrics are calculated.",
+  inputSchema: {
+    type: 'object',
+    properties: {
+      systemId: {
+        type: 'string',
+      },
+      config: {
+        type: 'object',
+        description: 'The configuration of the system version.',
+      },
+      name: {
+        type: 'string',
+        description: 'The name of the system version.',
+      },
+    },
+  },
+};
+
+export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { systemId, ...body } = args as any;
+  return asTextContentResult(await client.systems.versions.create(systemId, body));
+};
+
+export default { metadata, tool, handler };

package/src/tools/systems/versions/get-systems-versions.ts

@@ -0,0 +1,36 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'systems.versions',
+  operation: 'read',
+  tags: [],
+  httpMethod: 'get',
+  httpPath: '/systems/configs/{systemVersionId}',
+  operationId: 'getSystemVersion',
+};
+
+export const tool: Tool = {
+  name: 'get_systems_versions',
+  description: 'Retrieve a specific system version by ID.',
+  inputSchema: {
+    type: 'object',
+    properties: {
+      systemVersionId: {
+        type: 'string',
+      },
+    },
+  },
+};
+
+export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { systemVersionId, ...body } = args as any;
+  return asTextContentResult(await client.systems.versions.get(systemVersionId));
+};
+
+export default { metadata, tool, handler };