scorecard-ai-mcp 2.6.0 → 3.0.0

package/src/dynamic-tools.ts

@@ -1,159 +0,0 @@
-import Scorecard from 'scorecard-ai';
-import { Endpoint, asTextContentResult, ToolCallResult } from './tools/types';
-import { zodToJsonSchema } from 'zod-to-json-schema';
-import { z } from 'zod';
-import { Cabidela } from '@cloudflare/cabidela';
-
-function zodToInputSchema(schema: z.ZodSchema) {
-  return {
-    type: 'object' as const,
-    ...(zodToJsonSchema(schema) as any),
-  };
-}
-
-/**
- * A list of tools that expose all the endpoints in the API dynamically.
- *
- * Instead of exposing every endpoint as its own tool, which uses up too many tokens for LLMs to use at once,
- * we expose a single tool that can be used to search for endpoints by name, resource, operation, or tag, and then
- * a generic endpoint that can be used to invoke any endpoint with the provided arguments.
- *
- * @param endpoints - The endpoints to include in the list.
- */
-export function dynamicTools(endpoints: Endpoint[]): Endpoint[] {
-  const listEndpointsSchema = z.object({
-    search_query: z
-      .string()
-      .optional()
-      .describe(
-        'An optional search query to filter the endpoints by. Provide a partial name, resource, operation, or tag to filter the endpoints returned.',
-      ),
-  });
-
-  const listEndpointsTool = {
-    metadata: {
-      resource: 'dynamic_tools',
-      operation: 'read' as const,
-      tags: [],
-    },
-    tool: {
-      name: 'list_api_endpoints',
-      description: 'List or search for all endpoints in the Scorecard TypeScript API',
-      inputSchema: zodToInputSchema(listEndpointsSchema),
-    },
-    handler: async (
-      client: Scorecard,
-      args: Record<string, unknown> | undefined,
-    ): Promise<ToolCallResult> => {
-      const query = args && listEndpointsSchema.parse(args).search_query?.trim();
-
-      const filteredEndpoints =
-        query && query.length > 0 ?
-          endpoints.filter((endpoint) => {
-            const fieldsToMatch = [
-              endpoint.tool.name,
-              endpoint.tool.description,
-              endpoint.metadata.resource,
-              endpoint.metadata.operation,
-              ...endpoint.metadata.tags,
-            ];
-            return fieldsToMatch.some((field) => field && field.toLowerCase().includes(query.toLowerCase()));
-          })
-        : endpoints;
-
-      return asTextContentResult({
-        tools: filteredEndpoints.map(({ tool, metadata }) => ({
-          name: tool.name,
-          description: tool.description,
-          resource: metadata.resource,
-          operation: metadata.operation,
-          tags: metadata.tags,
-        })),
-      });
-    },
-  };
-
-  const getEndpointSchema = z.object({
-    endpoint: z.string().describe('The name of the endpoint to get the schema for.'),
-  });
-  const getEndpointTool = {
-    metadata: {
-      resource: 'dynamic_tools',
-      operation: 'read' as const,
-      tags: [],
-    },
-    tool: {
-      name: 'get_api_endpoint_schema',
-      description:
-        'Get the schema for an endpoint in the Scorecard TypeScript API. You can use the schema returned by this tool to invoke an endpoint with the `invoke_api_endpoint` tool.',
-      inputSchema: zodToInputSchema(getEndpointSchema),
-    },
-    handler: async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-      if (!args) {
-        throw new Error('No endpoint provided');
-      }
-      const endpointName = getEndpointSchema.parse(args).endpoint;
-
-      const endpoint = endpoints.find((e) => e.tool.name === endpointName);
-      if (!endpoint) {
-        throw new Error(`Endpoint ${endpointName} not found`);
-      }
-      return asTextContentResult(endpoint.tool);
-    },
-  };
-
-  const invokeEndpointSchema = z.object({
-    endpoint_name: z.string().describe('The name of the endpoint to invoke.'),
-    args: z
-      .record(z.string(), z.any())
-      .describe(
-        'The arguments to pass to the endpoint. This must match the schema returned by the `get_api_endpoint_schema` tool.',
-      ),
-  });
-
-  const invokeEndpointTool = {
-    metadata: {
-      resource: 'dynamic_tools',
-      operation: 'write' as const,
-      tags: [],
-    },
-    tool: {
-      name: 'invoke_api_endpoint',
-      description:
-        'Invoke an endpoint in the Scorecard TypeScript API. Note: use the `list_api_endpoints` tool to get the list of endpoints and `get_api_endpoint_schema` tool to get the schema for an endpoint.',
-      inputSchema: zodToInputSchema(invokeEndpointSchema),
-    },
-    handler: async (
-      client: Scorecard,
-      args: Record<string, unknown> | undefined,
-    ): Promise<ToolCallResult> => {
-      if (!args) {
-        throw new Error('No endpoint provided');
-      }
-      const { success, data, error } = invokeEndpointSchema.safeParse(args);
-      if (!success) {
-        throw new Error(`Invalid arguments for endpoint. ${error?.format()}`);
-      }
-      const { endpoint_name, args: endpointArgs } = data;
-
-      const endpoint = endpoints.find((e) => e.tool.name === endpoint_name);
-      if (!endpoint) {
-        throw new Error(
-          `Endpoint ${endpoint_name} not found. Use the \`list_api_endpoints\` tool to get the list of available endpoints.`,
-        );
-      }
-
-      try {
-        // Try to validate the arguments for a better error message
-        const cabidela = new Cabidela(endpoint.tool.inputSchema, { fullErrors: true });
-        cabidela.validate(endpointArgs);
-      } catch (error) {
-        throw new Error(`Invalid arguments for endpoint ${endpoint_name}:\n${error}`);
-      }
-
-      return await endpoint.handler(client, endpointArgs);
-    },
-  };
-
-  return [getEndpointTool, listEndpointsTool, invokeEndpointTool];
-}

package/src/filtering.ts

@@ -1,18 +0,0 @@
-// @ts-nocheck
-import initJq from 'jq-web';
-
-export async function maybeFilter(jqFilter: unknown | undefined, response: any): Promise<any> {
-  if (jqFilter && typeof jqFilter === 'string') {
-    return await jq(response, jqFilter);
-  } else {
-    return response;
-  }
-}
-
-async function jq(json: any, jqFilter: string) {
-  return (await initJq).json(json, jqFilter);
-}
-
-export function isJqError(error: any): error is Error {
-  return error instanceof Error && 'stderr' in error;
-}

package/src/tools/index.ts

@@ -1,131 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { Metadata, Endpoint, HandlerFunction } from './types';
-
-export { Metadata, Endpoint, HandlerFunction };
-
-import create_projects from './projects/create-projects';
-import list_projects from './projects/list-projects';
-import create_testsets from './testsets/create-testsets';
-import update_testsets from './testsets/update-testsets';
-import list_testsets from './testsets/list-testsets';
-import delete_testsets from './testsets/delete-testsets';
-import get_testsets from './testsets/get-testsets';
-import create_testcases from './testcases/create-testcases';
-import update_testcases from './testcases/update-testcases';
-import list_testcases from './testcases/list-testcases';
-import delete_testcases from './testcases/delete-testcases';
-import get_testcases from './testcases/get-testcases';
-import create_runs from './runs/create-runs';
-import list_runs from './runs/list-runs';
-import get_runs from './runs/get-runs';
-import create_metrics from './metrics/create-metrics';
-import update_metrics from './metrics/update-metrics';
-import list_metrics from './metrics/list-metrics';
-import delete_metrics from './metrics/delete-metrics';
-import get_metrics from './metrics/get-metrics';
-import create_records from './records/create-records';
-import list_records from './records/list-records';
-import delete_records from './records/delete-records';
-import upsert_scores from './scores/upsert-scores';
-import update_systems from './systems/update-systems';
-import list_systems from './systems/list-systems';
-import delete_systems from './systems/delete-systems';
-import get_systems from './systems/get-systems';
-import upsert_systems from './systems/upsert-systems';
-import get_systems_versions from './systems/versions/get-systems-versions';
-import upsert_systems_versions from './systems/versions/upsert-systems-versions';
-
-export const endpoints: Endpoint[] = [];
-
-function addEndpoint(endpoint: Endpoint) {
-  endpoints.push(endpoint);
-}
-
-addEndpoint(create_projects);
-addEndpoint(list_projects);
-addEndpoint(create_testsets);
-addEndpoint(update_testsets);
-addEndpoint(list_testsets);
-addEndpoint(delete_testsets);
-addEndpoint(get_testsets);
-addEndpoint(create_testcases);
-addEndpoint(update_testcases);
-addEndpoint(list_testcases);
-addEndpoint(delete_testcases);
-addEndpoint(get_testcases);
-addEndpoint(create_runs);
-addEndpoint(list_runs);
-addEndpoint(get_runs);
-addEndpoint(create_metrics);
-addEndpoint(update_metrics);
-addEndpoint(list_metrics);
-addEndpoint(delete_metrics);
-addEndpoint(get_metrics);
-addEndpoint(create_records);
-addEndpoint(list_records);
-addEndpoint(delete_records);
-addEndpoint(upsert_scores);
-addEndpoint(update_systems);
-addEndpoint(list_systems);
-addEndpoint(delete_systems);
-addEndpoint(get_systems);
-addEndpoint(upsert_systems);
-addEndpoint(get_systems_versions);
-addEndpoint(upsert_systems_versions);
-
-export type Filter = {
-  type: 'resource' | 'operation' | 'tag' | 'tool';
-  op: 'include' | 'exclude';
-  value: string;
-};
-
-export function query(filters: Filter[], endpoints: Endpoint[]): Endpoint[] {
-  const allExcludes = filters.length > 0 && filters.every((filter) => filter.op === 'exclude');
-  const unmatchedFilters = new Set(filters);
-
-  const filtered = endpoints.filter((endpoint: Endpoint) => {
-    let included = false || allExcludes;
-
-    for (const filter of filters) {
-      if (match(filter, endpoint)) {
-        unmatchedFilters.delete(filter);
-        included = filter.op === 'include';
-      }
-    }
-
-    return included;
-  });
-
-  // Check if any filters didn't match
-  const unmatched = Array.from(unmatchedFilters).filter((f) => f.type === 'tool' || f.type === 'resource');
-  if (unmatched.length > 0) {
-    throw new Error(
-      `The following filters did not match any endpoints: ${unmatched
-        .map((f) => `${f.type}=${f.value}`)
-        .join(', ')}`,
-    );
-  }
-
-  return filtered;
-}
-
-function match({ type, value }: Filter, endpoint: Endpoint): boolean {
-  switch (type) {
-    case 'resource': {
-      const regexStr = '^' + normalizeResource(value).replace(/\*/g, '.*') + '$';
-      const regex = new RegExp(regexStr);
-      return regex.test(normalizeResource(endpoint.metadata.resource));
-    }
-    case 'operation':
-      return endpoint.metadata.operation === value;
-    case 'tag':
-      return endpoint.metadata.tags.includes(value);
-    case 'tool':
-      return endpoint.tool.name === value;
-  }
-}
-
-function normalizeResource(resource: string): string {
-  return resource.toLowerCase().replace(/[^a-z.*\-_]*/g, '');
-}

package/src/tools/metrics/create-metrics.ts

@@ -1,383 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'metrics',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'post',
-  httpPath: '/projects/{projectId}/metrics',
-  operationId: 'createMetric',
-};
-
-export const tool: Tool = {
-  name: 'create_metrics',
-  description:
-    'Create a new Metric for evaluating system outputs. The structure of a metric depends on the evalType and outputType of the metric.',
-  inputSchema: {
-    type: 'object',
-    anyOf: [
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'AI-based evaluation type.',
-            enum: ['ai'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Integer output type.',
-            enum: ['int'],
-          },
-          promptTemplate: {
-            type: 'string',
-            description:
-              'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          evalModelName: {
-            type: 'string',
-            description: 'The AI model to use for evaluation.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for AI evaluation on how to score the metric.',
-          },
-          passingThreshold: {
-            type: 'integer',
-            description: 'The threshold for determining pass/fail from integer scores (1-5).',
-          },
-          temperature: {
-            type: 'number',
-            description: 'The temperature for AI evaluation (0-2).',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType', 'promptTemplate'],
-      },
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Human-based evaluation type.',
-            enum: ['human'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Integer output type.',
-            enum: ['int'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for human evaluators.',
-          },
-          passingThreshold: {
-            type: 'integer',
-            description: 'The threshold for determining pass/fail from integer scores (1-5).',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Heuristic-based evaluation type.',
-            enum: ['heuristic'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Integer output type.',
-            enum: ['int'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for heuristic evaluation logic.',
-          },
-          passingThreshold: {
-            type: 'integer',
-            description: 'The threshold for determining pass/fail from integer scores (1-5).',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'AI-based evaluation type.',
-            enum: ['ai'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Float output type (0-1).',
-            enum: ['float'],
-          },
-          promptTemplate: {
-            type: 'string',
-            description:
-              'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          evalModelName: {
-            type: 'string',
-            description: 'The AI model to use for evaluation.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for AI evaluation on how to score the metric.',
-          },
-          passingThreshold: {
-            type: 'number',
-            description: 'Threshold for determining pass/fail from float scores (0.0-1.0).',
-          },
-          temperature: {
-            type: 'number',
-            description: 'The temperature for AI evaluation (0-2).',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType', 'promptTemplate'],
-      },
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Human-based evaluation type.',
-            enum: ['human'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Float output type (0-1).',
-            enum: ['float'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for human evaluators.',
-          },
-          passingThreshold: {
-            type: 'number',
-            description: 'Threshold for determining pass/fail from float scores (0.0-1.0).',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Heuristic-based evaluation type.',
-            enum: ['heuristic'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Float output type (0-1).',
-            enum: ['float'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for heuristic evaluation logic.',
-          },
-          passingThreshold: {
-            type: 'number',
-            description: 'Threshold for determining pass/fail from float scores (0.0-1.0).',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'AI-based evaluation type.',
-            enum: ['ai'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Boolean output type.',
-            enum: ['boolean'],
-          },
-          promptTemplate: {
-            type: 'string',
-            description:
-              'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          evalModelName: {
-            type: 'string',
-            description: 'The AI model to use for evaluation.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for AI evaluation on how to score the metric.',
-          },
-          temperature: {
-            type: 'number',
-            description: 'The temperature for AI evaluation (0-2).',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType', 'promptTemplate'],
-      },
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Human-based evaluation type.',
-            enum: ['human'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Boolean output type.',
-            enum: ['boolean'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for human evaluators.',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          projectId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Heuristic-based evaluation type.',
-            enum: ['heuristic'],
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          outputType: {
-            type: 'string',
-            description: 'Boolean output type.',
-            enum: ['boolean'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for heuristic evaluation logic.',
-          },
-        },
-        required: ['projectId', 'evalType', 'name', 'outputType'],
-      },
-    ],
-  },
-  annotations: {},
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { projectId, ...body } = args as any;
-  try {
-    return asTextContentResult(await client.metrics.create(projectId, body));
-  } catch (error) {
-    if (error instanceof Scorecard.APIError) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };