scorecard-ai-mcp 2.5.0 → 3.0.0-beta.0

package/src/tools/records/list-records.ts

@@ -1,65 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'records',
-  operation: 'read',
-  tags: [],
-  httpMethod: 'get',
-  httpPath: '/runs/{runId}/records',
-  operationId: 'listRecords',
-};
-
-export const tool: Tool = {
-  name: 'list_records',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nRetrieve a paginated list of Records for a Run, including all scores for each record.\n\n# Response Schema\n```json\n{\n  type: 'object',\n  properties: {\n    data: {\n      type: 'array',\n      items: {\n        $ref: '#/$defs/record_list_response'\n      }\n    },\n    hasMore: {\n      type: 'boolean'\n    },\n    nextCursor: {\n      type: 'string'\n    },\n    total: {\n      type: 'integer'\n    }\n  },\n  required: [    'data',\n    'hasMore',\n    'nextCursor'\n  ],\n  $defs: {\n    record_list_response: {\n      allOf: [        {\n          $ref: '#/$defs/record'\n        }\n      ],\n      description: 'A record with all its associated scores.'\n    },\n    record: {\n      type: 'object',\n      description: 'A record of a system execution in the Scorecard system.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the Record.'\n        },\n        expected: {\n          type: 'object',\n          description: 'The expected outputs for the Testcase.',\n          additionalProperties: true\n        },\n        inputs: {\n          type: 'object',\n          description: 'The actual inputs sent to the system, which should match the system\\'s input schema.',\n          additionalProperties: true\n        },\n        outputs: {\n          type: 'object',\n          description: 'The actual outputs from the system.',\n          additionalProperties: true\n        },\n        runId: {\n          type: 'string',\n          description: 'The ID of the Run containing this Record.'\n        },\n        testcaseId: {\n          type: 'string',\n          description: 'The ID of the Testcase.'\n        }\n      },\n      required: [        'id',\n        'expected',\n        'inputs',\n        'outputs',\n        'runId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      runId: {
-        type: 'string',
-      },
-      cursor: {
-        type: 'string',
-        description:
-          'Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of results.',
-      },
-      limit: {
-        type: 'integer',
-        description:
-          'Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['runId'],
-  },
-  annotations: {
-    readOnlyHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { runId, jq_filter, ...body } = args as any;
-  const response = await client.records.list(runId, body).asResponse();
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await response.json()));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/runs/create-runs.ts

@@ -1,67 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'runs',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'post',
-  httpPath: '/projects/{projectId}/runs',
-  operationId: 'createRun',
-};
-
-export const tool: Tool = {
-  name: 'create_runs',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nCreate a new Run.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/run',\n  $defs: {\n    run: {\n      type: 'object',\n      description: 'A Run in the Scorecard system.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the Run.'\n        },\n        metricIds: {\n          type: 'array',\n          description: 'The IDs of the metrics this Run is using.',\n          items: {\n            type: 'string'\n          }\n        },\n        metricVersionIds: {\n          type: 'array',\n          description: 'The IDs of the metric versions this Run is using.',\n          items: {\n            type: 'string'\n          }\n        },\n        numExpectedRecords: {\n          type: 'number',\n          description: 'The number of expected records in the Run. Determined by the number of testcases in the Run\\'s Testset at the time of Run creation.'\n        },\n        numRecords: {\n          type: 'number',\n          description: 'The number of records in the Run.'\n        },\n        numScores: {\n          type: 'number',\n          description: 'The number of completed scores in the Run so far.'\n        },\n        status: {\n          type: 'string',\n          description: 'The status of the Run.',\n          enum: [            'pending',\n            'awaiting_execution',\n            'running_execution',\n            'awaiting_scoring',\n            'running_scoring',\n            'awaiting_human_scoring',\n            'completed'\n          ]\n        },\n        systemId: {\n          type: 'string',\n          description: 'The ID of the system this Run is using.'\n        },\n        systemVersionId: {\n          type: 'string',\n          description: 'The ID of the system version this Run is using.'\n        },\n        testsetId: {\n          type: 'string',\n          description: 'The ID of the Testset this Run is testing.'\n        }\n      },\n      required: [        'id',\n        'metricIds',\n        'metricVersionIds',\n        'numExpectedRecords',\n        'numRecords',\n        'numScores',\n        'status',\n        'systemId',\n        'systemVersionId',\n        'testsetId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      projectId: {
-        type: 'string',
-      },
-      metricIds: {
-        type: 'array',
-        description: 'The IDs of the metrics this Run is using.',
-        items: {
-          type: 'string',
-        },
-      },
-      systemVersionId: {
-        type: 'string',
-        description: 'The ID of the system version this Run is using.',
-      },
-      testsetId: {
-        type: 'string',
-        description: 'The ID of the Testset this Run is testing.',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['projectId', 'metricIds'],
-  },
-  annotations: {},
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { projectId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.runs.create(projectId, body)));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/runs/get-runs.ts

@@ -1,54 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'runs',
-  operation: 'read',
-  tags: [],
-  httpMethod: 'get',
-  httpPath: '/runs/{runId}',
-  operationId: 'getRun',
-};
-
-export const tool: Tool = {
-  name: 'get_runs',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nRetrieve a specific Run by ID.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/run',\n  $defs: {\n    run: {\n      type: 'object',\n      description: 'A Run in the Scorecard system.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the Run.'\n        },\n        metricIds: {\n          type: 'array',\n          description: 'The IDs of the metrics this Run is using.',\n          items: {\n            type: 'string'\n          }\n        },\n        metricVersionIds: {\n          type: 'array',\n          description: 'The IDs of the metric versions this Run is using.',\n          items: {\n            type: 'string'\n          }\n        },\n        numExpectedRecords: {\n          type: 'number',\n          description: 'The number of expected records in the Run. Determined by the number of testcases in the Run\\'s Testset at the time of Run creation.'\n        },\n        numRecords: {\n          type: 'number',\n          description: 'The number of records in the Run.'\n        },\n        numScores: {\n          type: 'number',\n          description: 'The number of completed scores in the Run so far.'\n        },\n        status: {\n          type: 'string',\n          description: 'The status of the Run.',\n          enum: [            'pending',\n            'awaiting_execution',\n            'running_execution',\n            'awaiting_scoring',\n            'running_scoring',\n            'awaiting_human_scoring',\n            'completed'\n          ]\n        },\n        systemId: {\n          type: 'string',\n          description: 'The ID of the system this Run is using.'\n        },\n        systemVersionId: {\n          type: 'string',\n          description: 'The ID of the system version this Run is using.'\n        },\n        testsetId: {\n          type: 'string',\n          description: 'The ID of the Testset this Run is testing.'\n        }\n      },\n      required: [        'id',\n        'metricIds',\n        'metricVersionIds',\n        'numExpectedRecords',\n        'numRecords',\n        'numScores',\n        'status',\n        'systemId',\n        'systemVersionId',\n        'testsetId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      runId: {
-        type: 'string',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['runId'],
-  },
-  annotations: {
-    readOnlyHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { runId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.runs.get(runId)));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/runs/list-runs.ts

@@ -1,65 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'runs',
-  operation: 'read',
-  tags: [],
-  httpMethod: 'get',
-  httpPath: '/projects/{projectId}/runs',
-  operationId: 'listRuns',
-};
-
-export const tool: Tool = {
-  name: 'list_runs',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nRetrieve a paginated list of all Runs for a Project. Runs are ordered by creation date, most recent first.\n\n# Response Schema\n```json\n{\n  type: 'object',\n  properties: {\n    data: {\n      type: 'array',\n      items: {\n        $ref: '#/$defs/run'\n      }\n    },\n    hasMore: {\n      type: 'boolean'\n    },\n    nextCursor: {\n      type: 'string'\n    },\n    total: {\n      type: 'integer'\n    }\n  },\n  required: [    'data',\n    'hasMore',\n    'nextCursor'\n  ],\n  $defs: {\n    run: {\n      type: 'object',\n      description: 'A Run in the Scorecard system.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the Run.'\n        },\n        metricIds: {\n          type: 'array',\n          description: 'The IDs of the metrics this Run is using.',\n          items: {\n            type: 'string'\n          }\n        },\n        metricVersionIds: {\n          type: 'array',\n          description: 'The IDs of the metric versions this Run is using.',\n          items: {\n            type: 'string'\n          }\n        },\n        numExpectedRecords: {\n          type: 'number',\n          description: 'The number of expected records in the Run. Determined by the number of testcases in the Run\\'s Testset at the time of Run creation.'\n        },\n        numRecords: {\n          type: 'number',\n          description: 'The number of records in the Run.'\n        },\n        numScores: {\n          type: 'number',\n          description: 'The number of completed scores in the Run so far.'\n        },\n        status: {\n          type: 'string',\n          description: 'The status of the Run.',\n          enum: [            'pending',\n            'awaiting_execution',\n            'running_execution',\n            'awaiting_scoring',\n            'running_scoring',\n            'awaiting_human_scoring',\n            'completed'\n          ]\n        },\n        systemId: {\n          type: 'string',\n          description: 'The ID of the system this Run is using.'\n        },\n        systemVersionId: {\n          type: 'string',\n          description: 'The ID of the system version this Run is using.'\n        },\n        testsetId: {\n          type: 'string',\n          description: 'The ID of the Testset this Run is testing.'\n        }\n      },\n      required: [        'id',\n        'metricIds',\n        'metricVersionIds',\n        'numExpectedRecords',\n        'numRecords',\n        'numScores',\n        'status',\n        'systemId',\n        'systemVersionId',\n        'testsetId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      projectId: {
-        type: 'string',
-      },
-      cursor: {
-        type: 'string',
-        description:
-          'Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of results.',
-      },
-      limit: {
-        type: 'integer',
-        description:
-          'Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['projectId'],
-  },
-  annotations: {
-    readOnlyHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { projectId, jq_filter, ...body } = args as any;
-  const response = await client.runs.list(projectId, body).asResponse();
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await response.json()));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/scores/upsert-scores.ts

@@ -1,65 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'scores',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'put',
-  httpPath: '/records/{recordId}/scores/{metricConfigId}',
-  operationId: 'upsertScore',
-};
-
-export const tool: Tool = {
-  name: 'upsert_scores',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nCreate or update a Score for a given Record and MetricConfig. If a Score with the specified Record ID and MetricConfig ID already exists, it will be updated. Otherwise, a new Score will be created. The score provided should conform to the schema defined by the MetricConfig; otherwise, validation errors will be reported.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/score',\n  $defs: {\n    score: {\n      type: 'object',\n      description: 'A Score represents the evaluation of a Record against a specific MetricConfig. The actual `score` is stored as flexible JSON. While any JSON is accepted, it is expected to conform to the output schema defined by the MetricConfig. Any discrepancies will be noted in the `validationErrors` field, but the Score will still be stored.',\n      properties: {\n        metricConfigId: {\n          type: 'string',\n          description: 'The ID of the MetricConfig this Score is for.'\n        },\n        recordId: {\n          type: 'string',\n          description: 'The ID of the Record this Score is for.'\n        },\n        score: {\n          type: 'object',\n          description: 'The score of the Record, as arbitrary JSON. This data should ideally conform to the output schema defined by the associated MetricConfig. If it doesn\\'t, validation errors will be captured in the `validationErrors` field.',\n          additionalProperties: true\n        },\n        validationErrors: {\n          type: 'array',\n          description: 'Validation errors found in the Score data. If present, the Score doesn\\'t fully conform to its MetricConfig\\'s schema.',\n          items: {\n            type: 'object',\n            properties: {\n              message: {\n                type: 'string',\n                description: 'Human-readable error description.'\n              },\n              path: {\n                type: 'string',\n                description: 'JSON Pointer to the field with the validation error.'\n              }\n            },\n            required: [              'message',\n              'path'\n            ]\n          }\n        }\n      },\n      required: [        'metricConfigId',\n        'recordId',\n        'score'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      recordId: {
-        type: 'string',
-      },
-      metricConfigId: {
-        type: 'string',
-      },
-      score: {
-        type: 'object',
-        description:
-          "The score of the Record, as arbitrary JSON. This data should ideally conform to the output schema defined by the associated MetricConfig. If it doesn't, validation errors will be captured in the `validationErrors` field.",
-        additionalProperties: true,
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['recordId', 'metricConfigId', 'score'],
-  },
-  annotations: {
-    idempotentHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { metricConfigId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(
-      await maybeFilter(jq_filter, await client.scores.upsert(metricConfigId, body)),
-    );
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/systems/delete-systems.ts

@@ -1,54 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'systems',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'delete',
-  httpPath: '/systems/{systemId}',
-  operationId: 'deleteSystem',
-};
-
-export const tool: Tool = {
-  name: 'delete_systems',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nDelete a system definition by ID. This will not delete associated system versions.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/system_delete_response',\n  $defs: {\n    system_delete_response: {\n      type: 'object',\n      properties: {\n        success: {\n          type: 'boolean',\n          description: 'Whether the deletion was successful.'\n        }\n      },\n      required: [        'success'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      systemId: {
-        type: 'string',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['systemId'],
-  },
-  annotations: {
-    idempotentHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { systemId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.systems.delete(systemId)));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/systems/get-systems.ts

@@ -1,54 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'systems',
-  operation: 'read',
-  tags: [],
-  httpMethod: 'get',
-  httpPath: '/systems/{systemId}',
-  operationId: 'getSystem',
-};
-
-export const tool: Tool = {
-  name: 'get_systems',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nRetrieve a specific system by ID.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/system',\n  $defs: {\n    system: {\n      type: 'object',\n      description: 'A System Under Test (SUT).\\n\\nSystems are templates - to run evaluations, pair them with a SystemVersion that provides specific\\nparameter values.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the system.'\n        },\n        description: {\n          type: 'string',\n          description: 'The description of the system.'\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the system. Unique within the project.'\n        },\n        productionVersion: {\n          $ref: '#/$defs/system_version'\n        },\n        versions: {\n          type: 'array',\n          description: 'The versions of the system.',\n          items: {\n            type: 'object',\n            description: 'A SystemVersion defines the specific settings for a System Under Test.\\n\\nSystem versions contain parameter values that determine system behavior during evaluation.\\nThey are immutable snapshots - once created, they never change.\\n\\nWhen running evaluations, you reference a specific systemVersionId to establish which system version to test.',\n            properties: {\n              id: {\n                type: 'string',\n                description: 'The ID of the system version.'\n              },\n              name: {\n                type: 'string',\n                description: 'The name of the system version.'\n              }\n            },\n            required: [              'id',\n              'name'\n            ]\n          }\n        }\n      },\n      required: [        'id',\n        'description',\n        'name',\n        'productionVersion',\n        'versions'\n      ]\n    },\n    system_version: {\n      type: 'object',\n      description: 'A SystemVersion defines the specific settings for a System Under Test.\\n\\nSystem versions contain parameter values that determine system behavior during evaluation.\\nThey are immutable snapshots - once created, they never change.\\n\\nWhen running evaluations, you reference a specific systemVersionId to establish which system version to test.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the system version.'\n        },\n        config: {\n          type: 'object',\n          description: 'The configuration of the system version.',\n          additionalProperties: true\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the system version.'\n        },\n        systemId: {\n          type: 'string',\n          description: 'The ID of the system the system version belongs to.'\n        }\n      },\n      required: [        'id',\n        'config',\n        'name',\n        'systemId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      systemId: {
-        type: 'string',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['systemId'],
-  },
-  annotations: {
-    readOnlyHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { systemId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.systems.get(systemId)));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/systems/list-systems.ts

@@ -1,65 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'systems',
-  operation: 'read',
-  tags: [],
-  httpMethod: 'get',
-  httpPath: '/projects/{projectId}/systems',
-  operationId: 'listSystems',
-};
-
-export const tool: Tool = {
-  name: 'list_systems',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nRetrieve a paginated list of all systems. Systems are ordered by creation date.\n\n# Response Schema\n```json\n{\n  type: 'object',\n  properties: {\n    data: {\n      type: 'array',\n      items: {\n        $ref: '#/$defs/system'\n      }\n    },\n    hasMore: {\n      type: 'boolean'\n    },\n    nextCursor: {\n      type: 'string'\n    },\n    total: {\n      type: 'integer'\n    }\n  },\n  required: [    'data',\n    'hasMore',\n    'nextCursor'\n  ],\n  $defs: {\n    system: {\n      type: 'object',\n      description: 'A System Under Test (SUT).\\n\\nSystems are templates - to run evaluations, pair them with a SystemVersion that provides specific\\nparameter values.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the system.'\n        },\n        description: {\n          type: 'string',\n          description: 'The description of the system.'\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the system. Unique within the project.'\n        },\n        productionVersion: {\n          $ref: '#/$defs/system_version'\n        },\n        versions: {\n          type: 'array',\n          description: 'The versions of the system.',\n          items: {\n            type: 'object',\n            description: 'A SystemVersion defines the specific settings for a System Under Test.\\n\\nSystem versions contain parameter values that determine system behavior during evaluation.\\nThey are immutable snapshots - once created, they never change.\\n\\nWhen running evaluations, you reference a specific systemVersionId to establish which system version to test.',\n            properties: {\n              id: {\n                type: 'string',\n                description: 'The ID of the system version.'\n              },\n              name: {\n                type: 'string',\n                description: 'The name of the system version.'\n              }\n            },\n            required: [              'id',\n              'name'\n            ]\n          }\n        }\n      },\n      required: [        'id',\n        'description',\n        'name',\n        'productionVersion',\n        'versions'\n      ]\n    },\n    system_version: {\n      type: 'object',\n      description: 'A SystemVersion defines the specific settings for a System Under Test.\\n\\nSystem versions contain parameter values that determine system behavior during evaluation.\\nThey are immutable snapshots - once created, they never change.\\n\\nWhen running evaluations, you reference a specific systemVersionId to establish which system version to test.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the system version.'\n        },\n        config: {\n          type: 'object',\n          description: 'The configuration of the system version.',\n          additionalProperties: true\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the system version.'\n        },\n        systemId: {\n          type: 'string',\n          description: 'The ID of the system the system version belongs to.'\n        }\n      },\n      required: [        'id',\n        'config',\n        'name',\n        'systemId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      projectId: {
-        type: 'string',
-      },
-      cursor: {
-        type: 'string',
-        description:
-          'Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of results.',
-      },
-      limit: {
-        type: 'integer',
-        description:
-          'Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['projectId'],
-  },
-  annotations: {
-    readOnlyHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { projectId, jq_filter, ...body } = args as any;
-  const response = await client.systems.list(projectId, body).asResponse();
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await response.json()));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/systems/update-systems.ts

@@ -1,64 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'systems',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'patch',
-  httpPath: '/systems/{systemId}',
-  operationId: 'updateSystem',
-};
-
-export const tool: Tool = {
-  name: 'update_systems',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nUpdate an existing system. Only the fields provided in the request body will be updated.\nIf a field is provided, the new content will replace the existing content.\nIf a field is not provided, the existing content will remain unchanged.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/system',\n  $defs: {\n    system: {\n      type: 'object',\n      description: 'A System Under Test (SUT).\\n\\nSystems are templates - to run evaluations, pair them with a SystemVersion that provides specific\\nparameter values.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the system.'\n        },\n        description: {\n          type: 'string',\n          description: 'The description of the system.'\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the system. Unique within the project.'\n        },\n        productionVersion: {\n          $ref: '#/$defs/system_version'\n        },\n        versions: {\n          type: 'array',\n          description: 'The versions of the system.',\n          items: {\n            type: 'object',\n            description: 'A SystemVersion defines the specific settings for a System Under Test.\\n\\nSystem versions contain parameter values that determine system behavior during evaluation.\\nThey are immutable snapshots - once created, they never change.\\n\\nWhen running evaluations, you reference a specific systemVersionId to establish which system version to test.',\n            properties: {\n              id: {\n                type: 'string',\n                description: 'The ID of the system version.'\n              },\n              name: {\n                type: 'string',\n                description: 'The name of the system version.'\n              }\n            },\n            required: [              'id',\n              'name'\n            ]\n          }\n        }\n      },\n      required: [        'id',\n        'description',\n        'name',\n        'productionVersion',\n        'versions'\n      ]\n    },\n    system_version: {\n      type: 'object',\n      description: 'A SystemVersion defines the specific settings for a System Under Test.\\n\\nSystem versions contain parameter values that determine system behavior during evaluation.\\nThey are immutable snapshots - once created, they never change.\\n\\nWhen running evaluations, you reference a specific systemVersionId to establish which system version to test.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the system version.'\n        },\n        config: {\n          type: 'object',\n          description: 'The configuration of the system version.',\n          additionalProperties: true\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the system version.'\n        },\n        systemId: {\n          type: 'string',\n          description: 'The ID of the system the system version belongs to.'\n        }\n      },\n      required: [        'id',\n        'config',\n        'name',\n        'systemId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      systemId: {
-        type: 'string',
-      },
-      description: {
-        type: 'string',
-        description: 'The description of the system.',
-      },
-      name: {
-        type: 'string',
-        description: 'The name of the system. Unique within the project.',
-      },
-      productionVersionId: {
-        type: 'string',
-        description: 'The ID of the production version of the system.',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['systemId'],
-  },
-  annotations: {},
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { systemId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.systems.update(systemId, body)));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };