scorecard-ai-mcp 2.5.0 → 3.0.0-beta.0

package/src/tools/index.ts

@@ -1,131 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { Metadata, Endpoint, HandlerFunction } from './types';
-
-export { Metadata, Endpoint, HandlerFunction };
-
-import create_projects from './projects/create-projects';
-import list_projects from './projects/list-projects';
-import create_testsets from './testsets/create-testsets';
-import update_testsets from './testsets/update-testsets';
-import list_testsets from './testsets/list-testsets';
-import delete_testsets from './testsets/delete-testsets';
-import get_testsets from './testsets/get-testsets';
-import create_testcases from './testcases/create-testcases';
-import update_testcases from './testcases/update-testcases';
-import list_testcases from './testcases/list-testcases';
-import delete_testcases from './testcases/delete-testcases';
-import get_testcases from './testcases/get-testcases';
-import create_runs from './runs/create-runs';
-import list_runs from './runs/list-runs';
-import get_runs from './runs/get-runs';
-import create_metrics from './metrics/create-metrics';
-import update_metrics from './metrics/update-metrics';
-import list_metrics from './metrics/list-metrics';
-import delete_metrics from './metrics/delete-metrics';
-import get_metrics from './metrics/get-metrics';
-import create_records from './records/create-records';
-import list_records from './records/list-records';
-import delete_records from './records/delete-records';
-import upsert_scores from './scores/upsert-scores';
-import update_systems from './systems/update-systems';
-import list_systems from './systems/list-systems';
-import delete_systems from './systems/delete-systems';
-import get_systems from './systems/get-systems';
-import upsert_systems from './systems/upsert-systems';
-import get_systems_versions from './systems/versions/get-systems-versions';
-import upsert_systems_versions from './systems/versions/upsert-systems-versions';
-
-export const endpoints: Endpoint[] = [];
-
-function addEndpoint(endpoint: Endpoint) {
-  endpoints.push(endpoint);
-}
-
-addEndpoint(create_projects);
-addEndpoint(list_projects);
-addEndpoint(create_testsets);
-addEndpoint(update_testsets);
-addEndpoint(list_testsets);
-addEndpoint(delete_testsets);
-addEndpoint(get_testsets);
-addEndpoint(create_testcases);
-addEndpoint(update_testcases);
-addEndpoint(list_testcases);
-addEndpoint(delete_testcases);
-addEndpoint(get_testcases);
-addEndpoint(create_runs);
-addEndpoint(list_runs);
-addEndpoint(get_runs);
-addEndpoint(create_metrics);
-addEndpoint(update_metrics);
-addEndpoint(list_metrics);
-addEndpoint(delete_metrics);
-addEndpoint(get_metrics);
-addEndpoint(create_records);
-addEndpoint(list_records);
-addEndpoint(delete_records);
-addEndpoint(upsert_scores);
-addEndpoint(update_systems);
-addEndpoint(list_systems);
-addEndpoint(delete_systems);
-addEndpoint(get_systems);
-addEndpoint(upsert_systems);
-addEndpoint(get_systems_versions);
-addEndpoint(upsert_systems_versions);
-
-export type Filter = {
-  type: 'resource' | 'operation' | 'tag' | 'tool';
-  op: 'include' | 'exclude';
-  value: string;
-};
-
-export function query(filters: Filter[], endpoints: Endpoint[]): Endpoint[] {
-  const allExcludes = filters.length > 0 && filters.every((filter) => filter.op === 'exclude');
-  const unmatchedFilters = new Set(filters);
-
-  const filtered = endpoints.filter((endpoint: Endpoint) => {
-    let included = false || allExcludes;
-
-    for (const filter of filters) {
-      if (match(filter, endpoint)) {
-        unmatchedFilters.delete(filter);
-        included = filter.op === 'include';
-      }
-    }
-
-    return included;
-  });
-
-  // Check if any filters didn't match
-  const unmatched = Array.from(unmatchedFilters).filter((f) => f.type === 'tool' || f.type === 'resource');
-  if (unmatched.length > 0) {
-    throw new Error(
-      `The following filters did not match any endpoints: ${unmatched
-        .map((f) => `${f.type}=${f.value}`)
-        .join(', ')}`,
-    );
-  }
-
-  return filtered;
-}
-
-function match({ type, value }: Filter, endpoint: Endpoint): boolean {
-  switch (type) {
-    case 'resource': {
-      const regexStr = '^' + normalizeResource(value).replace(/\*/g, '.*') + '$';
-      const regex = new RegExp(regexStr);
-      return regex.test(normalizeResource(endpoint.metadata.resource));
-    }
-    case 'operation':
-      return endpoint.metadata.operation === value;
-    case 'tag':
-      return endpoint.metadata.tags.includes(value);
-    case 'tool':
-      return endpoint.tool.name === value;
-  }
-}
-
-function normalizeResource(resource: string): string {
-  return resource.toLowerCase().replace(/[^a-z.*\-_]*/g, '');
-}

package/src/tools/metrics/create-metrics.ts

@@ -1,376 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { Metadata, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'metrics',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'post',
-  httpPath: '/projects/{projectId}/metrics',
-  operationId: 'createMetric',
-};
-
-export const tool: Tool = {
-  name: 'create_metrics',
-  description:
-    'Create a new Metric for evaluating system outputs. The structure of a metric depends on the evalType and outputType of the metric.',
-  inputSchema: {
-    type: 'object',
-    anyOf: [
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'AI-based evaluation type.',
-            enum: ['ai'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Integer output type.',
-            enum: ['int'],
-          },
-          promptTemplate: {
-            type: 'string',
-            description:
-              'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          evalModelName: {
-            type: 'string',
-            description: 'The AI model to use for evaluation.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for AI evaluation on how to score the metric.',
-          },
-          passingThreshold: {
-            type: 'integer',
-            description: 'The threshold for determining pass/fail from integer scores (1-5).',
-          },
-          temperature: {
-            type: 'number',
-            description: 'The temperature for AI evaluation (0-2).',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType', 'promptTemplate'],
-      },
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Human-based evaluation type.',
-            enum: ['human'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Integer output type.',
-            enum: ['int'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for human evaluators.',
-          },
-          passingThreshold: {
-            type: 'integer',
-            description: 'The threshold for determining pass/fail from integer scores (1-5).',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Heuristic-based evaluation type.',
-            enum: ['heuristic'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Integer output type.',
-            enum: ['int'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for heuristic evaluation logic.',
-          },
-          passingThreshold: {
-            type: 'integer',
-            description: 'The threshold for determining pass/fail from integer scores (1-5).',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'AI-based evaluation type.',
-            enum: ['ai'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Float output type (0-1).',
-            enum: ['float'],
-          },
-          promptTemplate: {
-            type: 'string',
-            description:
-              'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          evalModelName: {
-            type: 'string',
-            description: 'The AI model to use for evaluation.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for AI evaluation on how to score the metric.',
-          },
-          passingThreshold: {
-            type: 'number',
-            description: 'Threshold for determining pass/fail from float scores (0.0-1.0).',
-          },
-          temperature: {
-            type: 'number',
-            description: 'The temperature for AI evaluation (0-2).',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType', 'promptTemplate'],
-      },
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Human-based evaluation type.',
-            enum: ['human'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Float output type (0-1).',
-            enum: ['float'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for human evaluators.',
-          },
-          passingThreshold: {
-            type: 'number',
-            description: 'Threshold for determining pass/fail from float scores (0.0-1.0).',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Heuristic-based evaluation type.',
-            enum: ['heuristic'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Float output type (0-1).',
-            enum: ['float'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for heuristic evaluation logic.',
-          },
-          passingThreshold: {
-            type: 'number',
-            description: 'Threshold for determining pass/fail from float scores (0.0-1.0).',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'AI-based evaluation type.',
-            enum: ['ai'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Boolean output type.',
-            enum: ['boolean'],
-          },
-          promptTemplate: {
-            type: 'string',
-            description:
-              'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          evalModelName: {
-            type: 'string',
-            description: 'The AI model to use for evaluation.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for AI evaluation on how to score the metric.',
-          },
-          temperature: {
-            type: 'number',
-            description: 'The temperature for AI evaluation (0-2).',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType', 'promptTemplate'],
-      },
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Human-based evaluation type.',
-            enum: ['human'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Boolean output type.',
-            enum: ['boolean'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for human evaluators.',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Heuristic-based evaluation type.',
-            enum: ['heuristic'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Boolean output type.',
-            enum: ['boolean'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for heuristic evaluation logic.',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType'],
-      },
-    ],
-  },
-  annotations: {},
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { projectId, ...body } = args as any;
-  return asTextContentResult(await client.metrics.create(projectId, body));
-};
-
-export default { metadata, tool, handler };

package/src/tools/metrics/delete-metrics.ts

@@ -1,54 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'metrics',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'delete',
-  httpPath: '/metrics/{metricId}',
-  operationId: 'deleteMetric',
-};
-
-export const tool: Tool = {
-  name: 'delete_metrics',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nDelete a specific Metric by ID. The metric will be removed from metric groups and monitors.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/metric_delete_response',\n  $defs: {\n    metric_delete_response: {\n      type: 'object',\n      properties: {\n        success: {\n          type: 'boolean',\n          description: 'Whether the deletion was successful.'\n        }\n      },\n      required: [        'success'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      metricId: {
-        type: 'string',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['metricId'],
-  },
-  annotations: {
-    idempotentHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { metricId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.metrics.delete(metricId)));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/metrics/get-metrics.ts

@@ -1,39 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { Metadata, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'metrics',
-  operation: 'read',
-  tags: [],
-  httpMethod: 'get',
-  httpPath: '/metrics/{metricId}',
-  operationId: 'getMetric',
-};
-
-export const tool: Tool = {
-  name: 'get_metrics',
-  description: 'Retrieve a specific Metric by ID.',
-  inputSchema: {
-    type: 'object',
-    properties: {
-      metricId: {
-        type: 'string',
-      },
-    },
-    required: ['metricId'],
-  },
-  annotations: {
-    readOnlyHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { metricId, ...body } = args as any;
-  return asTextContentResult(await client.metrics.get(metricId));
-};
-
-export default { metadata, tool, handler };

package/src/tools/metrics/list-metrics.ts

@@ -1,51 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { Metadata, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'metrics',
-  operation: 'read',
-  tags: [],
-  httpMethod: 'get',
-  httpPath: '/projects/{projectId}/metrics',
-  operationId: 'listMetrics',
-};
-
-export const tool: Tool = {
-  name: 'list_metrics',
-  description:
-    'List Metrics configured for the specified Project. Metrics are returned in reverse chronological order.',
-  inputSchema: {
-    type: 'object',
-    properties: {
-      projectId: {
-        type: 'string',
-      },
-      cursor: {
-        type: 'string',
-        description:
-          'Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of results.',
-      },
-      limit: {
-        type: 'integer',
-        description:
-          'Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.',
-      },
-    },
-    required: ['projectId'],
-  },
-  annotations: {
-    readOnlyHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { projectId, ...body } = args as any;
-  const response = await client.metrics.list(projectId, body).asResponse();
-  return asTextContentResult(await response.json());
-};
-
-export default { metadata, tool, handler };