scorecard-ai-mcp 1.0.0-alpha.9 → 1.1.0

package/src/tools/index.ts

@@ -17,6 +17,8 @@ import list_testcases from './testcases/list-testcases';
 import delete_testcases from './testcases/delete-testcases';
 import get_testcases from './testcases/get-testcases';
 import create_runs from './runs/create-runs';
+import create_metrics from './metrics/create-metrics';
+import update_metrics from './metrics/update-metrics';
 import create_records from './records/create-records';
 import upsert_scores from './scores/upsert-scores';
 import create_systems from './systems/create-systems';
@@ -24,9 +26,9 @@ import update_systems from './systems/update-systems';
 import list_systems from './systems/list-systems';
 import delete_systems from './systems/delete-systems';
 import get_systems from './systems/get-systems';
-import create_system_configs from './system-configs/create-system-configs';
-import list_system_configs from './system-configs/list-system-configs';
-import get_system_configs from './system-configs/get-system-configs';
+import create_systems_versions from './systems/versions/create-systems-versions';
+import list_systems_versions from './systems/versions/list-systems-versions';
+import get_systems_versions from './systems/versions/get-systems-versions';
 
 export const endpoints: Endpoint[] = [];
 
@@ -47,6 +49,8 @@ addEndpoint(list_testcases);
 addEndpoint(delete_testcases);
 addEndpoint(get_testcases);
 addEndpoint(create_runs);
+addEndpoint(create_metrics);
+addEndpoint(update_metrics);
 addEndpoint(create_records);
 addEndpoint(upsert_scores);
 addEndpoint(create_systems);
@@ -54,9 +58,9 @@ addEndpoint(update_systems);
 addEndpoint(list_systems);
 addEndpoint(delete_systems);
 addEndpoint(get_systems);
-addEndpoint(create_system_configs);
-addEndpoint(list_system_configs);
-addEndpoint(get_system_configs);
+addEndpoint(create_systems_versions);
+addEndpoint(list_systems_versions);
+addEndpoint(get_systems_versions);
 
 export type Filter = {
   type: 'resource' | 'operation' | 'tag' | 'tool';
@@ -82,9 +86,10 @@ export function query(filters: Filter[], endpoints: Endpoint[]): Endpoint[] {
   });
 
   // Check if any filters didn't match
-  if (unmatchedFilters.size > 0) {
+  const unmatched = Array.from(unmatchedFilters).filter((f) => f.type === 'tool' || f.type === 'resource');
+  if (unmatched.length > 0) {
     throw new Error(
-      `The following filters did not match any endpoints: ${[...unmatchedFilters]
+      `The following filters did not match any endpoints: ${unmatched
         .map((f) => `${f.type}=${f.value}`)
         .join(', ')}`,
     );

package/src/tools/metrics/create-metrics.ts

@@ -0,0 +1,252 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'metrics',
+  operation: 'write',
+  tags: [],
+  httpMethod: 'post',
+  httpPath: '/projects/{projectId}/metrics',
+  operationId: 'createMetric',
+};
+
+export const tool: Tool = {
+  name: 'create_metrics',
+  description:
+    'Create a new Metric for evaluating system outputs. The structure of a metric depends on the evalType and outputType of the metric.',
+  inputSchema: {
+    type: 'object',
+    anyOf: [
+      {
+        type: 'object',
+        properties: {
+          projectId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'AI-based evaluation type.',
+            enum: ['ai'],
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+          outputType: {
+            type: 'string',
+            description: 'Integer output type.',
+            enum: ['int'],
+          },
+          promptTemplate: {
+            type: 'string',
+            description:
+              'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          evalModelName: {
+            type: 'string',
+            description: 'The AI model to use for evaluation.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Guidelines for AI evaluation on how to score the metric.',
+          },
+          passingThreshold: {
+            type: 'integer',
+            description: 'The threshold for determining pass/fail from integer scores (1-5).',
+          },
+          temperature: {
+            type: 'number',
+            description: 'The temperature for AI evaluation (0-2).',
+          },
+        },
+      },
+      {
+        type: 'object',
+        properties: {
+          projectId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'Human-based evaluation type.',
+            enum: ['human'],
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+          outputType: {
+            type: 'string',
+            description: 'Integer output type.',
+            enum: ['int'],
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Guidelines for human evaluators.',
+          },
+          passingThreshold: {
+            type: 'integer',
+            description: 'The threshold for determining pass/fail from integer scores (1-5).',
+          },
+        },
+      },
+      {
+        type: 'object',
+        properties: {
+          projectId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'Heuristic-based evaluation type.',
+            enum: ['heuristic'],
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+          outputType: {
+            type: 'string',
+            description: 'Integer output type.',
+            enum: ['int'],
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Optional guidelines for heuristic evaluation logic.',
+          },
+          passingThreshold: {
+            type: 'integer',
+            description: 'The threshold for determining pass/fail from integer scores (1-5).',
+          },
+        },
+      },
+      {
+        type: 'object',
+        properties: {
+          projectId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'AI-based evaluation type.',
+            enum: ['ai'],
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+          outputType: {
+            type: 'string',
+            description: 'Boolean output type.',
+            enum: ['boolean'],
+          },
+          promptTemplate: {
+            type: 'string',
+            description:
+              'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          evalModelName: {
+            type: 'string',
+            description: 'The AI model to use for evaluation.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Guidelines for AI evaluation on how to score the metric.',
+          },
+          temperature: {
+            type: 'number',
+            description: 'The temperature for AI evaluation (0-2).',
+          },
+        },
+      },
+      {
+        type: 'object',
+        properties: {
+          projectId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'Human-based evaluation type.',
+            enum: ['human'],
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+          outputType: {
+            type: 'string',
+            description: 'Boolean output type.',
+            enum: ['boolean'],
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Guidelines for human evaluators.',
+          },
+        },
+      },
+      {
+        type: 'object',
+        properties: {
+          projectId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'Heuristic-based evaluation type.',
+            enum: ['heuristic'],
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+          outputType: {
+            type: 'string',
+            description: 'Boolean output type.',
+            enum: ['boolean'],
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Optional guidelines for heuristic evaluation logic.',
+          },
+        },
+      },
+    ],
+  },
+};
+
+export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { projectId, ...body } = args as any;
+  return asTextContentResult(await client.metrics.create(projectId, body));
+};
+
+export default { metadata, tool, handler };

package/src/tools/metrics/update-metrics.ts

@@ -0,0 +1,252 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'metrics',
+  operation: 'write',
+  tags: [],
+  httpMethod: 'patch',
+  httpPath: '/metrics/{metricId}',
+  operationId: 'updateMetric',
+};
+
+export const tool: Tool = {
+  name: 'update_metrics',
+  description:
+    'Update an existing Metric. You must specify the evalType and outputType of the metric. The structure of a metric depends on the evalType and outputType of the metric.',
+  inputSchema: {
+    type: 'object',
+    anyOf: [
+      {
+        type: 'object',
+        properties: {
+          metricId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'AI-based evaluation type.',
+            enum: ['ai'],
+          },
+          outputType: {
+            type: 'string',
+            description: 'Integer output type.',
+            enum: ['int'],
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          evalModelName: {
+            type: 'string',
+            description: 'The AI model to use for evaluation.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Guidelines for AI evaluation on how to score the metric.',
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+          passingThreshold: {
+            type: 'integer',
+            description: 'The threshold for determining pass/fail from integer scores (1-5).',
+          },
+          promptTemplate: {
+            type: 'string',
+            description:
+              'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
+          },
+          temperature: {
+            type: 'number',
+            description: 'The temperature for AI evaluation (0-2).',
+          },
+        },
+      },
+      {
+        type: 'object',
+        properties: {
+          metricId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'Human-based evaluation type.',
+            enum: ['human'],
+          },
+          outputType: {
+            type: 'string',
+            description: 'Integer output type.',
+            enum: ['int'],
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Guidelines for human evaluators.',
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+          passingThreshold: {
+            type: 'integer',
+            description: 'The threshold for determining pass/fail from integer scores (1-5).',
+          },
+        },
+      },
+      {
+        type: 'object',
+        properties: {
+          metricId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'Heuristic-based evaluation type.',
+            enum: ['heuristic'],
+          },
+          outputType: {
+            type: 'string',
+            description: 'Integer output type.',
+            enum: ['int'],
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Optional guidelines for heuristic evaluation logic.',
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+          passingThreshold: {
+            type: 'integer',
+            description: 'The threshold for determining pass/fail from integer scores (1-5).',
+          },
+        },
+      },
+      {
+        type: 'object',
+        properties: {
+          metricId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'AI-based evaluation type.',
+            enum: ['ai'],
+          },
+          outputType: {
+            type: 'string',
+            description: 'Boolean output type.',
+            enum: ['boolean'],
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          evalModelName: {
+            type: 'string',
+            description: 'The AI model to use for evaluation.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Guidelines for AI evaluation on how to score the metric.',
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+          promptTemplate: {
+            type: 'string',
+            description:
+              'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
+          },
+          temperature: {
+            type: 'number',
+            description: 'The temperature for AI evaluation (0-2).',
+          },
+        },
+      },
+      {
+        type: 'object',
+        properties: {
+          metricId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'Human-based evaluation type.',
+            enum: ['human'],
+          },
+          outputType: {
+            type: 'string',
+            description: 'Boolean output type.',
+            enum: ['boolean'],
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Guidelines for human evaluators.',
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+        },
+      },
+      {
+        type: 'object',
+        properties: {
+          metricId: {
+            type: 'string',
+          },
+          evalType: {
+            type: 'string',
+            description: 'Heuristic-based evaluation type.',
+            enum: ['heuristic'],
+          },
+          outputType: {
+            type: 'string',
+            description: 'Boolean output type.',
+            enum: ['boolean'],
+          },
+          description: {
+            type: 'string',
+            description: 'The description of the Metric.',
+          },
+          guidelines: {
+            type: 'string',
+            description: 'Optional guidelines for heuristic evaluation logic.',
+          },
+          name: {
+            type: 'string',
+            description: 'The name of the Metric.',
+          },
+        },
+      },
+    ],
+  },
+};
+
+export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { metricId, ...body } = args as any;
+  return asTextContentResult(await client.metrics.update(metricId, body));
+};
+
+export default { metadata, tool, handler };

package/src/tools/runs/create-runs.ts

@@ -31,9 +31,9 @@ export const tool: Tool = {
           type: 'string',
         },
       },
-      systemConfigId: {
+      systemVersionId: {
         type: 'string',
-        description: 'The ID of the system configuration this Run is using.',
+        description: 'The ID of the system version this Run is using.',
       },
       testsetId: {
         type: 'string',

package/src/tools/systems/delete-systems.ts

@@ -17,7 +17,7 @@ export const metadata: Metadata = {
 
 export const tool: Tool = {
   name: 'delete_systems',
-  description: 'Delete a system definition by ID. This will not delete associated system configurations.',
+  description: 'Delete a system definition by ID. This will not delete associated system versions.',
   inputSchema: {
     type: 'object',
     properties: {

package/src/tools/systems/versions/create-systems-versions.ts

@@ -0,0 +1,45 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'systems.versions',
+  operation: 'write',
+  tags: [],
+  httpMethod: 'post',
+  httpPath: '/systems/{systemId}/configs',
+  operationId: 'createSystemVersion',
+};
+
+export const tool: Tool = {
+  name: 'create_systems_versions',
+  description:
+    "Create a new version for a system.\n\nEach version contains specific parameter values that match the system's `configSchema` - things like model parameters, thresholds, or processing options.\nOnce created, versions cannot be modified, ensuring stable reference points for evaluations.\n\nWhen creating a system version:\n- The `config` object is validated against the parent system's `configSchema`.\n- System versions with validation errors are still stored, with errors included in the response.\n- Validation errors indicate fields that don't match the schema but don't prevent creation.\n- Having validation errors may affect how some evaluation metrics are calculated.",
+  inputSchema: {
+    type: 'object',
+    properties: {
+      systemId: {
+        type: 'string',
+      },
+      config: {
+        type: 'object',
+        description: 'The configuration of the system version.',
+      },
+      name: {
+        type: 'string',
+        description: 'The name of the system version.',
+      },
+    },
+  },
+};
+
+export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { systemId, ...body } = args as any;
+  return asTextContentResult(await client.systems.versions.create(systemId, body));
+};
+
+export default { metadata, tool, handler };

package/src/tools/{system-configs/get-system-configs.ts → systems/versions/get-systems-versions.ts}

@@ -3,25 +3,25 @@
 import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
 
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import type { Metadata } from '../';
+import type { Metadata } from '../../';
 import Scorecard from 'scorecard-ai';
 
 export const metadata: Metadata = {
-  resource: 'system_configs',
+  resource: 'systems.versions',
   operation: 'read',
   tags: [],
   httpMethod: 'get',
-  httpPath: '/systems/configs/{systemConfigId}',
-  operationId: 'getSystemConfig',
+  httpPath: '/systems/configs/{systemVersionId}',
+  operationId: 'getSystemVersion',
 };
 
 export const tool: Tool = {
-  name: 'get_system_configs',
-  description: 'Retrieve a specific system configuration by ID.',
+  name: 'get_systems_versions',
+  description: 'Retrieve a specific system version by ID.',
   inputSchema: {
     type: 'object',
     properties: {
-      systemConfigId: {
+      systemVersionId: {
         type: 'string',
       },
     },
@@ -29,8 +29,8 @@ export const tool: Tool = {
 };
 
 export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { systemConfigId, ...body } = args as any;
-  return asTextContentResult(await client.systemConfigs.get(systemConfigId));
+  const { systemVersionId, ...body } = args as any;
+  return asTextContentResult(await client.systems.versions.get(systemVersionId));
 };
 
 export default { metadata, tool, handler };