scorecard-ai-mcp 2.6.0 → 3.0.0

package/src/tools/systems/update-systems.ts

@@ -1,64 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'systems',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'patch',
-  httpPath: '/systems/{systemId}',
-  operationId: 'updateSystem',
-};
-
-export const tool: Tool = {
-  name: 'update_systems',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nUpdate an existing system. Only the fields provided in the request body will be updated.\nIf a field is provided, the new content will replace the existing content.\nIf a field is not provided, the existing content will remain unchanged.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/system',\n  $defs: {\n    system: {\n      type: 'object',\n      description: 'A System Under Test (SUT).\\n\\nSystems are templates - to run evaluations, pair them with a SystemVersion that provides specific\\nparameter values.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the system.'\n        },\n        description: {\n          type: 'string',\n          description: 'The description of the system.'\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the system. Unique within the project.'\n        },\n        productionVersion: {\n          $ref: '#/$defs/system_version'\n        },\n        versions: {\n          type: 'array',\n          description: 'The versions of the system.',\n          items: {\n            type: 'object',\n            description: 'A SystemVersion defines the specific settings for a System Under Test.\\n\\nSystem versions contain parameter values that determine system behavior during evaluation.\\nThey are immutable snapshots - once created, they never change.\\n\\nWhen running evaluations, you reference a specific systemVersionId to establish which system version to test.',\n            properties: {\n              id: {\n                type: 'string',\n                description: 'The ID of the system version.'\n              },\n              name: {\n                type: 'string',\n                description: 'The name of the system version.'\n              }\n            },\n            required: [              'id',\n              'name'\n            ]\n          }\n        }\n      },\n      required: [        'id',\n        'description',\n        'name',\n        'productionVersion',\n        'versions'\n      ]\n    },\n    system_version: {\n      type: 'object',\n      description: 'A SystemVersion defines the specific settings for a System Under Test.\\n\\nSystem versions contain parameter values that determine system behavior during evaluation.\\nThey are immutable snapshots - once created, they never change.\\n\\nWhen running evaluations, you reference a specific systemVersionId to establish which system version to test.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the system version.'\n        },\n        config: {\n          type: 'object',\n          description: 'The configuration of the system version.',\n          additionalProperties: true\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the system version.'\n        },\n        systemId: {\n          type: 'string',\n          description: 'The ID of the system the system version belongs to.'\n        }\n      },\n      required: [        'id',\n        'config',\n        'name',\n        'systemId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      systemId: {
-        type: 'string',
-      },
-      description: {
-        type: 'string',
-        description: 'The description of the system.',
-      },
-      name: {
-        type: 'string',
-        description: 'The name of the system. Unique within the project.',
-      },
-      productionVersionId: {
-        type: 'string',
-        description: 'The ID of the production version of the system.',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['systemId'],
-  },
-  annotations: {},
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { systemId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.systems.update(systemId, body)));
-  } catch (error) {
-    if (error instanceof Scorecard.APIError || isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/systems/upsert-systems.ts

@@ -1,66 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'systems',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'post',
-  httpPath: '/projects/{projectId}/systems',
-  operationId: 'upsertSystem',
-};
-
-export const tool: Tool = {
-  name: 'upsert_systems',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nCreate a new system. If one with the same name in the project exists, it updates it instead.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/system',\n  $defs: {\n    system: {\n      type: 'object',\n      description: 'A System Under Test (SUT).\\n\\nSystems are templates - to run evaluations, pair them with a SystemVersion that provides specific\\nparameter values.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the system.'\n        },\n        description: {\n          type: 'string',\n          description: 'The description of the system.'\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the system. Unique within the project.'\n        },\n        productionVersion: {\n          $ref: '#/$defs/system_version'\n        },\n        versions: {\n          type: 'array',\n          description: 'The versions of the system.',\n          items: {\n            type: 'object',\n            description: 'A SystemVersion defines the specific settings for a System Under Test.\\n\\nSystem versions contain parameter values that determine system behavior during evaluation.\\nThey are immutable snapshots - once created, they never change.\\n\\nWhen running evaluations, you reference a specific systemVersionId to establish which system version to test.',\n            properties: {\n              id: {\n                type: 'string',\n                description: 'The ID of the system version.'\n              },\n              name: {\n                type: 'string',\n                description: 'The name of the system version.'\n              }\n            },\n            required: [              'id',\n              'name'\n            ]\n          }\n        }\n      },\n      required: [        'id',\n        'description',\n        'name',\n        'productionVersion',\n        'versions'\n      ]\n    },\n    system_version: {\n      type: 'object',\n      description: 'A SystemVersion defines the specific settings for a System Under Test.\\n\\nSystem versions contain parameter values that determine system behavior during evaluation.\\nThey are immutable snapshots - once created, they never change.\\n\\nWhen running evaluations, you reference a specific systemVersionId to establish which system version to test.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the system version.'\n        },\n        config: {\n          type: 'object',\n          description: 'The configuration of the system version.',\n          additionalProperties: true\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the system version.'\n        },\n        systemId: {\n          type: 'string',\n          description: 'The ID of the system the system version belongs to.'\n        }\n      },\n      required: [        'id',\n        'config',\n        'name',\n        'systemId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      projectId: {
-        type: 'string',
-      },
-      config: {
-        type: 'object',
-        description: 'The configuration of the system.',
-        additionalProperties: true,
-      },
-      description: {
-        type: 'string',
-        description: 'The description of the system.',
-      },
-      name: {
-        type: 'string',
-        description:
-          'The name of the system. Should be unique within the project. Default is "Default system"',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['projectId', 'config'],
-  },
-  annotations: {},
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { projectId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.systems.upsert(projectId, body)));
-  } catch (error) {
-    if (error instanceof Scorecard.APIError || isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/systems/versions/get-systems-versions.ts

@@ -1,56 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'systems.versions',
-  operation: 'read',
-  tags: [],
-  httpMethod: 'get',
-  httpPath: '/systems/versions/{systemVersionId}',
-  operationId: 'getSystemVersion',
-};
-
-export const tool: Tool = {
-  name: 'get_systems_versions',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nRetrieve a specific system version by ID.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/system_version',\n  $defs: {\n    system_version: {\n      type: 'object',\n      description: 'A SystemVersion defines the specific settings for a System Under Test.\\n\\nSystem versions contain parameter values that determine system behavior during evaluation.\\nThey are immutable snapshots - once created, they never change.\\n\\nWhen running evaluations, you reference a specific systemVersionId to establish which system version to test.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the system version.'\n        },\n        config: {\n          type: 'object',\n          description: 'The configuration of the system version.',\n          additionalProperties: true\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the system version.'\n        },\n        systemId: {\n          type: 'string',\n          description: 'The ID of the system the system version belongs to.'\n        }\n      },\n      required: [        'id',\n        'config',\n        'name',\n        'systemId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      systemVersionId: {
-        type: 'string',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['systemVersionId'],
-  },
-  annotations: {
-    readOnlyHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { systemVersionId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(
-      await maybeFilter(jq_filter, await client.systems.versions.get(systemVersionId)),
-    );
-  } catch (error) {
-    if (error instanceof Scorecard.APIError || isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/systems/versions/upsert-systems-versions.ts

@@ -1,64 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'systems.versions',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'post',
-  httpPath: '/systems/{systemId}/versions',
-  operationId: 'upsertSystemVersion',
-};
-
-export const tool: Tool = {
-  name: 'upsert_systems_versions',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nCreate a new system version if it does not already exist. Does **not** set the created version to be the system's production version.\n\nIf there is already a system version with the same config, its name will be updated.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/system_version',\n  $defs: {\n    system_version: {\n      type: 'object',\n      description: 'A SystemVersion defines the specific settings for a System Under Test.\\n\\nSystem versions contain parameter values that determine system behavior during evaluation.\\nThey are immutable snapshots - once created, they never change.\\n\\nWhen running evaluations, you reference a specific systemVersionId to establish which system version to test.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the system version.'\n        },\n        config: {\n          type: 'object',\n          description: 'The configuration of the system version.',\n          additionalProperties: true\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the system version.'\n        },\n        systemId: {\n          type: 'string',\n          description: 'The ID of the system the system version belongs to.'\n        }\n      },\n      required: [        'id',\n        'config',\n        'name',\n        'systemId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      systemId: {
-        type: 'string',
-      },
-      config: {
-        type: 'object',
-        description: 'The configuration of the system version.',
-        additionalProperties: true,
-      },
-      name: {
-        type: 'string',
-        description:
-          "The name of the system version. If creating a new system version and the name isn't provided, it will be autogenerated.",
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['systemId', 'config'],
-  },
-  annotations: {},
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { systemId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(
-      await maybeFilter(jq_filter, await client.systems.versions.upsert(systemId, body)),
-    );
-  } catch (error) {
-    if (error instanceof Scorecard.APIError || isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/testcases/create-testcases.ts

@@ -1,67 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'testcases',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'post',
-  httpPath: '/testsets/{testsetId}/testcases',
-  operationId: 'createTestcases',
-};
-
-export const tool: Tool = {
-  name: 'create_testcases',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nCreate multiple Testcases in the specified Testset.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/testcase_create_response',\n  $defs: {\n    testcase_create_response: {\n      type: 'object',\n      properties: {\n        items: {\n          type: 'array',\n          items: {\n            $ref: '#/$defs/testcase'\n          }\n        }\n      },\n      required: [        'items'\n      ]\n    },\n    testcase: {\n      type: 'object',\n      description: 'A test case in the Scorecard system. Contains JSON data that is validated against the schema defined by its Testset.\\nThe `inputs` and `expected` fields are derived from the `data` field based on the Testset\\'s `fieldMapping`, and include all mapped fields, including those with validation errors.\\nTestcases are stored regardless of validation results, with any validation errors included in the `validationErrors` field.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the Testcase.'\n        },\n        expected: {\n          type: 'object',\n          description: 'Derived from data based on the Testset\\'s fieldMapping. Contains all fields marked as expected outputs, including those with validation errors.',\n          additionalProperties: true\n        },\n        inputs: {\n          type: 'object',\n          description: 'Derived from data based on the Testset\\'s fieldMapping. Contains all fields marked as inputs, including those with validation errors.',\n          additionalProperties: true\n        },\n        jsonData: {\n          type: 'object',\n          description: 'The JSON data of the Testcase, which is validated against the Testset\\'s schema.',\n          additionalProperties: true\n        },\n        testsetId: {\n          type: 'string',\n          description: 'The ID of the Testset this Testcase belongs to.'\n        },\n        validationErrors: {\n          type: 'array',\n          description: 'Validation errors found in the Testcase data. If present, the Testcase doesn\\'t fully conform to its Testset\\'s schema.',\n          items: {\n            type: 'object',\n            properties: {\n              message: {\n                type: 'string',\n                description: 'Human-readable error description.'\n              },\n              path: {\n                type: 'string',\n                description: 'JSON Pointer to the field with the validation error.'\n              }\n            },\n            required: [              'message',\n              'path'\n            ]\n          }\n        }\n      },\n      required: [        'id',\n        'expected',\n        'inputs',\n        'jsonData',\n        'testsetId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      testsetId: {
-        type: 'string',
-      },
-      items: {
-        type: 'array',
-        description: 'Testcases to create (max 100).',
-        items: {
-          type: 'object',
-          properties: {
-            jsonData: {
-              type: 'object',
-              description: "The JSON data of the Testcase, which is validated against the Testset's schema.",
-              additionalProperties: true,
-            },
-          },
-          required: ['jsonData'],
-        },
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['testsetId', 'items'],
-  },
-  annotations: {},
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { testsetId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.testcases.create(testsetId, body)));
-  } catch (error) {
-    if (error instanceof Scorecard.APIError || isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/testcases/delete-testcases.ts

@@ -1,56 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'testcases',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'post',
-  httpPath: '/testcases/bulk-delete',
-  operationId: 'deleteTestcases',
-};
-
-export const tool: Tool = {
-  name: 'delete_testcases',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nDelete multiple Testcases by their IDs.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/testcase_delete_response',\n  $defs: {\n    testcase_delete_response: {\n      type: 'object',\n      properties: {\n        success: {\n          type: 'boolean',\n          description: 'Whether the deletion was successful.'\n        }\n      },\n      required: [        'success'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      ids: {
-        type: 'array',
-        description: 'IDs of Testcases to delete.',
-        items: {
-          type: 'string',
-        },
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['ids'],
-  },
-  annotations: {},
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.testcases.delete(body)));
-  } catch (error) {
-    if (error instanceof Scorecard.APIError || isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/testcases/get-testcases.ts

@@ -1,54 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'testcases',
-  operation: 'read',
-  tags: [],
-  httpMethod: 'get',
-  httpPath: '/testcases/{testcaseId}',
-  operationId: 'getTestcase',
-};
-
-export const tool: Tool = {
-  name: 'get_testcases',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nRetrieve a specific Testcase by ID.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/testcase',\n  $defs: {\n    testcase: {\n      type: 'object',\n      description: 'A test case in the Scorecard system. Contains JSON data that is validated against the schema defined by its Testset.\\nThe `inputs` and `expected` fields are derived from the `data` field based on the Testset\\'s `fieldMapping`, and include all mapped fields, including those with validation errors.\\nTestcases are stored regardless of validation results, with any validation errors included in the `validationErrors` field.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the Testcase.'\n        },\n        expected: {\n          type: 'object',\n          description: 'Derived from data based on the Testset\\'s fieldMapping. Contains all fields marked as expected outputs, including those with validation errors.',\n          additionalProperties: true\n        },\n        inputs: {\n          type: 'object',\n          description: 'Derived from data based on the Testset\\'s fieldMapping. Contains all fields marked as inputs, including those with validation errors.',\n          additionalProperties: true\n        },\n        jsonData: {\n          type: 'object',\n          description: 'The JSON data of the Testcase, which is validated against the Testset\\'s schema.',\n          additionalProperties: true\n        },\n        testsetId: {\n          type: 'string',\n          description: 'The ID of the Testset this Testcase belongs to.'\n        },\n        validationErrors: {\n          type: 'array',\n          description: 'Validation errors found in the Testcase data. If present, the Testcase doesn\\'t fully conform to its Testset\\'s schema.',\n          items: {\n            type: 'object',\n            properties: {\n              message: {\n                type: 'string',\n                description: 'Human-readable error description.'\n              },\n              path: {\n                type: 'string',\n                description: 'JSON Pointer to the field with the validation error.'\n              }\n            },\n            required: [              'message',\n              'path'\n            ]\n          }\n        }\n      },\n      required: [        'id',\n        'expected',\n        'inputs',\n        'jsonData',\n        'testsetId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      testcaseId: {
-        type: 'string',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['testcaseId'],
-  },
-  annotations: {
-    readOnlyHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { testcaseId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.testcases.get(testcaseId)));
-  } catch (error) {
-    if (error instanceof Scorecard.APIError || isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/testcases/list-testcases.ts

@@ -1,65 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'testcases',
-  operation: 'read',
-  tags: [],
-  httpMethod: 'get',
-  httpPath: '/testsets/{testsetId}/testcases',
-  operationId: 'listTestcases',
-};
-
-export const tool: Tool = {
-  name: 'list_testcases',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nRetrieve a paginated list of Testcases belonging to a Testset.\n\n# Response Schema\n```json\n{\n  type: 'object',\n  properties: {\n    data: {\n      type: 'array',\n      items: {\n        $ref: '#/$defs/testcase'\n      }\n    },\n    hasMore: {\n      type: 'boolean'\n    },\n    nextCursor: {\n      type: 'string'\n    },\n    total: {\n      type: 'integer'\n    }\n  },\n  required: [    'data',\n    'hasMore',\n    'nextCursor'\n  ],\n  $defs: {\n    testcase: {\n      type: 'object',\n      description: 'A test case in the Scorecard system. Contains JSON data that is validated against the schema defined by its Testset.\\nThe `inputs` and `expected` fields are derived from the `data` field based on the Testset\\'s `fieldMapping`, and include all mapped fields, including those with validation errors.\\nTestcases are stored regardless of validation results, with any validation errors included in the `validationErrors` field.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the Testcase.'\n        },\n        expected: {\n          type: 'object',\n          description: 'Derived from data based on the Testset\\'s fieldMapping. Contains all fields marked as expected outputs, including those with validation errors.',\n          additionalProperties: true\n        },\n        inputs: {\n          type: 'object',\n          description: 'Derived from data based on the Testset\\'s fieldMapping. Contains all fields marked as inputs, including those with validation errors.',\n          additionalProperties: true\n        },\n        jsonData: {\n          type: 'object',\n          description: 'The JSON data of the Testcase, which is validated against the Testset\\'s schema.',\n          additionalProperties: true\n        },\n        testsetId: {\n          type: 'string',\n          description: 'The ID of the Testset this Testcase belongs to.'\n        },\n        validationErrors: {\n          type: 'array',\n          description: 'Validation errors found in the Testcase data. If present, the Testcase doesn\\'t fully conform to its Testset\\'s schema.',\n          items: {\n            type: 'object',\n            properties: {\n              message: {\n                type: 'string',\n                description: 'Human-readable error description.'\n              },\n              path: {\n                type: 'string',\n                description: 'JSON Pointer to the field with the validation error.'\n              }\n            },\n            required: [              'message',\n              'path'\n            ]\n          }\n        }\n      },\n      required: [        'id',\n        'expected',\n        'inputs',\n        'jsonData',\n        'testsetId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      testsetId: {
-        type: 'string',
-      },
-      cursor: {
-        type: 'string',
-        description:
-          'Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of results.',
-      },
-      limit: {
-        type: 'integer',
-        description:
-          'Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['testsetId'],
-  },
-  annotations: {
-    readOnlyHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { testsetId, jq_filter, ...body } = args as any;
-  const response = await client.testcases.list(testsetId, body).asResponse();
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await response.json()));
-  } catch (error) {
-    if (error instanceof Scorecard.APIError || isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/testcases/update-testcases.ts

@@ -1,59 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'testcases',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'put',
-  httpPath: '/testcases/{testcaseId}',
-  operationId: 'updateTestcase',
-};
-
-export const tool: Tool = {
-  name: 'update_testcases',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nReplace the data of an existing Testcase while keeping its ID.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/testcase',\n  $defs: {\n    testcase: {\n      type: 'object',\n      description: 'A test case in the Scorecard system. Contains JSON data that is validated against the schema defined by its Testset.\\nThe `inputs` and `expected` fields are derived from the `data` field based on the Testset\\'s `fieldMapping`, and include all mapped fields, including those with validation errors.\\nTestcases are stored regardless of validation results, with any validation errors included in the `validationErrors` field.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the Testcase.'\n        },\n        expected: {\n          type: 'object',\n          description: 'Derived from data based on the Testset\\'s fieldMapping. Contains all fields marked as expected outputs, including those with validation errors.',\n          additionalProperties: true\n        },\n        inputs: {\n          type: 'object',\n          description: 'Derived from data based on the Testset\\'s fieldMapping. Contains all fields marked as inputs, including those with validation errors.',\n          additionalProperties: true\n        },\n        jsonData: {\n          type: 'object',\n          description: 'The JSON data of the Testcase, which is validated against the Testset\\'s schema.',\n          additionalProperties: true\n        },\n        testsetId: {\n          type: 'string',\n          description: 'The ID of the Testset this Testcase belongs to.'\n        },\n        validationErrors: {\n          type: 'array',\n          description: 'Validation errors found in the Testcase data. If present, the Testcase doesn\\'t fully conform to its Testset\\'s schema.',\n          items: {\n            type: 'object',\n            properties: {\n              message: {\n                type: 'string',\n                description: 'Human-readable error description.'\n              },\n              path: {\n                type: 'string',\n                description: 'JSON Pointer to the field with the validation error.'\n              }\n            },\n            required: [              'message',\n              'path'\n            ]\n          }\n        }\n      },\n      required: [        'id',\n        'expected',\n        'inputs',\n        'jsonData',\n        'testsetId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      testcaseId: {
-        type: 'string',
-      },
-      jsonData: {
-        type: 'object',
-        description: "The JSON data of the Testcase, which is validated against the Testset's schema.",
-        additionalProperties: true,
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['testcaseId', 'jsonData'],
-  },
-  annotations: {
-    idempotentHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { testcaseId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.testcases.update(testcaseId, body)));
-  } catch (error) {
-    if (error instanceof Scorecard.APIError || isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };