scorecard-ai-mcp 2.5.0 → 3.0.0-beta.0

package/src/tools/metrics/update-metrics.ts

@@ -1,376 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { Metadata, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'metrics',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'patch',
-  httpPath: '/metrics/{metricId}',
-  operationId: 'updateMetric',
-};
-
-export const tool: Tool = {
-  name: 'update_metrics',
-  description:
-    'Update an existing Metric. You must specify the evalType and outputType of the metric. The structure of a metric depends on the evalType and outputType of the metric.',
-  inputSchema: {
-    type: 'object',
-    anyOf: [
-      {
-        type: 'object',
-        properties: {
-          metricId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'AI-based evaluation type.',
-            enum: ['ai'],
-          },
-          outputType: {
-            type: 'string',
-            description: 'Integer output type.',
-            enum: ['int'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          evalModelName: {
-            type: 'string',
-            description: 'The AI model to use for evaluation.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for AI evaluation on how to score the metric.',
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          passingThreshold: {
-            type: 'integer',
-            description: 'The threshold for determining pass/fail from integer scores (1-5).',
-          },
-          promptTemplate: {
-            type: 'string',
-            description:
-              'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
-          },
-          temperature: {
-            type: 'number',
-            description: 'The temperature for AI evaluation (0-2).',
-          },
-        },
-        required: ['metricId', 'evalType', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          metricId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Human-based evaluation type.',
-            enum: ['human'],
-          },
-          outputType: {
-            type: 'string',
-            description: 'Integer output type.',
-            enum: ['int'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for human evaluators.',
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          passingThreshold: {
-            type: 'integer',
-            description: 'The threshold for determining pass/fail from integer scores (1-5).',
-          },
-        },
-        required: ['metricId', 'evalType', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          metricId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Heuristic-based evaluation type.',
-            enum: ['heuristic'],
-          },
-          outputType: {
-            type: 'string',
-            description: 'Integer output type.',
-            enum: ['int'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for heuristic evaluation logic.',
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          passingThreshold: {
-            type: 'integer',
-            description: 'The threshold for determining pass/fail from integer scores (1-5).',
-          },
-        },
-        required: ['metricId', 'evalType', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          metricId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'AI-based evaluation type.',
-            enum: ['ai'],
-          },
-          outputType: {
-            type: 'string',
-            description: 'Float output type (0-1).',
-            enum: ['float'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          evalModelName: {
-            type: 'string',
-            description: 'The AI model to use for evaluation.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for AI evaluation on how to score the metric.',
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          passingThreshold: {
-            type: 'number',
-            description: 'Threshold for determining pass/fail from float scores (0.0-1.0).',
-          },
-          promptTemplate: {
-            type: 'string',
-            description:
-              'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
-          },
-          temperature: {
-            type: 'number',
-            description: 'The temperature for AI evaluation (0-2).',
-          },
-        },
-        required: ['metricId', 'evalType', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          metricId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Human-based evaluation type.',
-            enum: ['human'],
-          },
-          outputType: {
-            type: 'string',
-            description: 'Float output type (0-1).',
-            enum: ['float'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for human evaluators.',
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          passingThreshold: {
-            type: 'number',
-            description: 'Threshold for determining pass/fail from float scores (0.0-1.0).',
-          },
-        },
-        required: ['metricId', 'evalType', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          metricId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Heuristic-based evaluation type.',
-            enum: ['heuristic'],
-          },
-          outputType: {
-            type: 'string',
-            description: 'Float output type (0-1).',
-            enum: ['float'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for heuristic evaluation logic.',
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          passingThreshold: {
-            type: 'number',
-            description: 'Threshold for determining pass/fail from float scores (0.0-1.0).',
-          },
-        },
-        required: ['metricId', 'evalType', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          metricId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'AI-based evaluation type.',
-            enum: ['ai'],
-          },
-          outputType: {
-            type: 'string',
-            description: 'Boolean output type.',
-            enum: ['boolean'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          evalModelName: {
-            type: 'string',
-            description: 'The AI model to use for evaluation.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for AI evaluation on how to score the metric.',
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-          promptTemplate: {
-            type: 'string',
-            description:
-              'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
-          },
-          temperature: {
-            type: 'number',
-            description: 'The temperature for AI evaluation (0-2).',
-          },
-        },
-        required: ['metricId', 'evalType', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          metricId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Human-based evaluation type.',
-            enum: ['human'],
-          },
-          outputType: {
-            type: 'string',
-            description: 'Boolean output type.',
-            enum: ['boolean'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for human evaluators.',
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-        },
-        required: ['metricId', 'evalType', 'outputType'],
-      },
-      {
-        type: 'object',
-        properties: {
-          metricId: {
-            type: 'string',
-          },
-          evalType: {
-            type: 'string',
-            description: 'Heuristic-based evaluation type.',
-            enum: ['heuristic'],
-          },
-          outputType: {
-            type: 'string',
-            description: 'Boolean output type.',
-            enum: ['boolean'],
-          },
-          description: {
-            type: 'string',
-            description: 'The description of the Metric.',
-          },
-          guidelines: {
-            type: 'string',
-            description: 'Guidelines for heuristic evaluation logic.',
-          },
-          name: {
-            type: 'string',
-            description: 'The name of the Metric.',
-          },
-        },
-        required: ['metricId', 'evalType', 'outputType'],
-      },
-    ],
-  },
-  annotations: {},
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { metricId, ...body } = args as any;
-  return asTextContentResult(await client.metrics.update(metricId, body));
-};
-
-export default { metadata, tool, handler };

package/src/tools/projects/create-projects.ts

@@ -1,57 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'projects',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'post',
-  httpPath: '/projects',
-  operationId: 'createProject',
-};
-
-export const tool: Tool = {
-  name: 'create_projects',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nCreate a new Project.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/project',\n  $defs: {\n    project: {\n      type: 'object',\n      description: 'A Project in the Scorecard system.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the Project.'\n        },\n        description: {\n          type: 'string',\n          description: 'The description of the Project.'\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the Project.'\n        }\n      },\n      required: [        'id',\n        'description',\n        'name'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      description: {
-        type: 'string',
-        description: 'The description of the Project.',
-      },
-      name: {
-        type: 'string',
-        description: 'The name of the Project.',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['description', 'name'],
-  },
-  annotations: {},
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.projects.create(body)));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/projects/list-projects.ts

@@ -1,62 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'projects',
-  operation: 'read',
-  tags: [],
-  httpMethod: 'get',
-  httpPath: '/projects',
-  operationId: 'listProjects',
-};
-
-export const tool: Tool = {
-  name: 'list_projects',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nRetrieve a paginated list of all Projects. Projects are ordered by creation date, with oldest Projects first.\n\n# Response Schema\n```json\n{\n  type: 'object',\n  properties: {\n    data: {\n      type: 'array',\n      items: {\n        $ref: '#/$defs/project'\n      }\n    },\n    hasMore: {\n      type: 'boolean'\n    },\n    nextCursor: {\n      type: 'string'\n    },\n    total: {\n      type: 'integer'\n    }\n  },\n  required: [    'data',\n    'hasMore',\n    'nextCursor'\n  ],\n  $defs: {\n    project: {\n      type: 'object',\n      description: 'A Project in the Scorecard system.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the Project.'\n        },\n        description: {\n          type: 'string',\n          description: 'The description of the Project.'\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the Project.'\n        }\n      },\n      required: [        'id',\n        'description',\n        'name'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      cursor: {
-        type: 'string',
-        description:
-          'Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of results.',
-      },
-      limit: {
-        type: 'integer',
-        description:
-          'Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: [],
-  },
-  annotations: {
-    readOnlyHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { jq_filter, ...body } = args as any;
-  const response = await client.projects.list(body).asResponse();
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await response.json()));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/records/create-records.ts

@@ -1,71 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'records',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'post',
-  httpPath: '/runs/{runId}/records',
-  operationId: 'createRecord',
-};
-
-export const tool: Tool = {
-  name: 'create_records',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nCreate a new Record in a Run.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/record',\n  $defs: {\n    record: {\n      type: 'object',\n      description: 'A record of a system execution in the Scorecard system.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the Record.'\n        },\n        expected: {\n          type: 'object',\n          description: 'The expected outputs for the Testcase.',\n          additionalProperties: true\n        },\n        inputs: {\n          type: 'object',\n          description: 'The actual inputs sent to the system, which should match the system\\'s input schema.',\n          additionalProperties: true\n        },\n        outputs: {\n          type: 'object',\n          description: 'The actual outputs from the system.',\n          additionalProperties: true\n        },\n        runId: {\n          type: 'string',\n          description: 'The ID of the Run containing this Record.'\n        },\n        testcaseId: {\n          type: 'string',\n          description: 'The ID of the Testcase.'\n        }\n      },\n      required: [        'id',\n        'expected',\n        'inputs',\n        'outputs',\n        'runId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      runId: {
-        type: 'string',
-      },
-      expected: {
-        type: 'object',
-        description: 'The expected outputs for the Testcase.',
-        additionalProperties: true,
-      },
-      inputs: {
-        type: 'object',
-        description: "The actual inputs sent to the system, which should match the system's input schema.",
-        additionalProperties: true,
-      },
-      outputs: {
-        type: 'object',
-        description: 'The actual outputs from the system.',
-        additionalProperties: true,
-      },
-      testcaseId: {
-        type: 'string',
-        description: 'The ID of the Testcase.',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['runId', 'expected', 'inputs', 'outputs'],
-  },
-  annotations: {},
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { runId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.records.create(runId, body)));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/records/delete-records.ts

@@ -1,54 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'records',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'delete',
-  httpPath: '/records/{recordId}',
-  operationId: 'deleteRecord',
-};
-
-export const tool: Tool = {
-  name: 'delete_records',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nDelete a specific Record by ID.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/record_delete_response',\n  $defs: {\n    record_delete_response: {\n      type: 'object',\n      properties: {\n        success: {\n          type: 'boolean',\n          description: 'Whether the deletion was successful.'\n        }\n      },\n      required: [        'success'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      recordId: {
-        type: 'string',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['recordId'],
-  },
-  annotations: {
-    idempotentHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { recordId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.records.delete(recordId)));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };