scorecard-ai-mcp 2.5.0 → 3.0.0-beta.0

package/src/tools/systems/upsert-systems.ts

@@ -1,66 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'systems',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'post',
-  httpPath: '/projects/{projectId}/systems',
-  operationId: 'upsertSystem',
-};
-
-export const tool: Tool = {
-  name: 'upsert_systems',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nCreate a new system. If one with the same name in the project exists, it updates it instead.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/system',\n  $defs: {\n    system: {\n      type: 'object',\n      description: 'A System Under Test (SUT).\\n\\nSystems are templates - to run evaluations, pair them with a SystemVersion that provides specific\\nparameter values.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the system.'\n        },\n        description: {\n          type: 'string',\n          description: 'The description of the system.'\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the system. Unique within the project.'\n        },\n        productionVersion: {\n          $ref: '#/$defs/system_version'\n        },\n        versions: {\n          type: 'array',\n          description: 'The versions of the system.',\n          items: {\n            type: 'object',\n            description: 'A SystemVersion defines the specific settings for a System Under Test.\\n\\nSystem versions contain parameter values that determine system behavior during evaluation.\\nThey are immutable snapshots - once created, they never change.\\n\\nWhen running evaluations, you reference a specific systemVersionId to establish which system version to test.',\n            properties: {\n              id: {\n                type: 'string',\n                description: 'The ID of the system version.'\n              },\n              name: {\n                type: 'string',\n                description: 'The name of the system version.'\n              }\n            },\n            required: [              'id',\n              'name'\n            ]\n          }\n        }\n      },\n      required: [        'id',\n        'description',\n        'name',\n        'productionVersion',\n        'versions'\n      ]\n    },\n    system_version: {\n      type: 'object',\n      description: 'A SystemVersion defines the specific settings for a System Under Test.\\n\\nSystem versions contain parameter values that determine system behavior during evaluation.\\nThey are immutable snapshots - once created, they never change.\\n\\nWhen running evaluations, you reference a specific systemVersionId to establish which system version to test.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the system version.'\n        },\n        config: {\n          type: 'object',\n          description: 'The configuration of the system version.',\n          additionalProperties: true\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the system version.'\n        },\n        systemId: {\n          type: 'string',\n          description: 'The ID of the system the system version belongs to.'\n        }\n      },\n      required: [        'id',\n        'config',\n        'name',\n        'systemId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      projectId: {
-        type: 'string',
-      },
-      config: {
-        type: 'object',
-        description: 'The configuration of the system.',
-        additionalProperties: true,
-      },
-      description: {
-        type: 'string',
-        description: 'The description of the system.',
-      },
-      name: {
-        type: 'string',
-        description:
-          'The name of the system. Should be unique within the project. Default is "Default system"',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['projectId', 'config'],
-  },
-  annotations: {},
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { projectId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.systems.upsert(projectId, body)));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/systems/versions/get-systems-versions.ts

@@ -1,56 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'systems.versions',
-  operation: 'read',
-  tags: [],
-  httpMethod: 'get',
-  httpPath: '/systems/versions/{systemVersionId}',
-  operationId: 'getSystemVersion',
-};
-
-export const tool: Tool = {
-  name: 'get_systems_versions',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nRetrieve a specific system version by ID.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/system_version',\n  $defs: {\n    system_version: {\n      type: 'object',\n      description: 'A SystemVersion defines the specific settings for a System Under Test.\\n\\nSystem versions contain parameter values that determine system behavior during evaluation.\\nThey are immutable snapshots - once created, they never change.\\n\\nWhen running evaluations, you reference a specific systemVersionId to establish which system version to test.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the system version.'\n        },\n        config: {\n          type: 'object',\n          description: 'The configuration of the system version.',\n          additionalProperties: true\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the system version.'\n        },\n        systemId: {\n          type: 'string',\n          description: 'The ID of the system the system version belongs to.'\n        }\n      },\n      required: [        'id',\n        'config',\n        'name',\n        'systemId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      systemVersionId: {
-        type: 'string',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['systemVersionId'],
-  },
-  annotations: {
-    readOnlyHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { systemVersionId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(
-      await maybeFilter(jq_filter, await client.systems.versions.get(systemVersionId)),
-    );
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/systems/versions/upsert-systems-versions.ts

@@ -1,64 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'systems.versions',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'post',
-  httpPath: '/systems/{systemId}/versions',
-  operationId: 'upsertSystemVersion',
-};
-
-export const tool: Tool = {
-  name: 'upsert_systems_versions',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nCreate a new system version if it does not already exist. Does **not** set the created version to be the system's production version.\n\nIf there is already a system version with the same config, its name will be updated.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/system_version',\n  $defs: {\n    system_version: {\n      type: 'object',\n      description: 'A SystemVersion defines the specific settings for a System Under Test.\\n\\nSystem versions contain parameter values that determine system behavior during evaluation.\\nThey are immutable snapshots - once created, they never change.\\n\\nWhen running evaluations, you reference a specific systemVersionId to establish which system version to test.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the system version.'\n        },\n        config: {\n          type: 'object',\n          description: 'The configuration of the system version.',\n          additionalProperties: true\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the system version.'\n        },\n        systemId: {\n          type: 'string',\n          description: 'The ID of the system the system version belongs to.'\n        }\n      },\n      required: [        'id',\n        'config',\n        'name',\n        'systemId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      systemId: {
-        type: 'string',
-      },
-      config: {
-        type: 'object',
-        description: 'The configuration of the system version.',
-        additionalProperties: true,
-      },
-      name: {
-        type: 'string',
-        description:
-          "The name of the system version. If creating a new system version and the name isn't provided, it will be autogenerated.",
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['systemId', 'config'],
-  },
-  annotations: {},
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { systemId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(
-      await maybeFilter(jq_filter, await client.systems.versions.upsert(systemId, body)),
-    );
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/testcases/create-testcases.ts

@@ -1,67 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'testcases',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'post',
-  httpPath: '/testsets/{testsetId}/testcases',
-  operationId: 'createTestcases',
-};
-
-export const tool: Tool = {
-  name: 'create_testcases',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nCreate multiple Testcases in the specified Testset.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/testcase_create_response',\n  $defs: {\n    testcase_create_response: {\n      type: 'object',\n      properties: {\n        items: {\n          type: 'array',\n          items: {\n            $ref: '#/$defs/testcase'\n          }\n        }\n      },\n      required: [        'items'\n      ]\n    },\n    testcase: {\n      type: 'object',\n      description: 'A test case in the Scorecard system. Contains JSON data that is validated against the schema defined by its Testset.\\nThe `inputs` and `expected` fields are derived from the `data` field based on the Testset\\'s `fieldMapping`, and include all mapped fields, including those with validation errors.\\nTestcases are stored regardless of validation results, with any validation errors included in the `validationErrors` field.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the Testcase.'\n        },\n        expected: {\n          type: 'object',\n          description: 'Derived from data based on the Testset\\'s fieldMapping. Contains all fields marked as expected outputs, including those with validation errors.',\n          additionalProperties: true\n        },\n        inputs: {\n          type: 'object',\n          description: 'Derived from data based on the Testset\\'s fieldMapping. Contains all fields marked as inputs, including those with validation errors.',\n          additionalProperties: true\n        },\n        jsonData: {\n          type: 'object',\n          description: 'The JSON data of the Testcase, which is validated against the Testset\\'s schema.',\n          additionalProperties: true\n        },\n        testsetId: {\n          type: 'string',\n          description: 'The ID of the Testset this Testcase belongs to.'\n        },\n        validationErrors: {\n          type: 'array',\n          description: 'Validation errors found in the Testcase data. If present, the Testcase doesn\\'t fully conform to its Testset\\'s schema.',\n          items: {\n            type: 'object',\n            properties: {\n              message: {\n                type: 'string',\n                description: 'Human-readable error description.'\n              },\n              path: {\n                type: 'string',\n                description: 'JSON Pointer to the field with the validation error.'\n              }\n            },\n            required: [              'message',\n              'path'\n            ]\n          }\n        }\n      },\n      required: [        'id',\n        'expected',\n        'inputs',\n        'jsonData',\n        'testsetId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      testsetId: {
-        type: 'string',
-      },
-      items: {
-        type: 'array',
-        description: 'Testcases to create (max 100).',
-        items: {
-          type: 'object',
-          properties: {
-            jsonData: {
-              type: 'object',
-              description: "The JSON data of the Testcase, which is validated against the Testset's schema.",
-              additionalProperties: true,
-            },
-          },
-          required: ['jsonData'],
-        },
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['testsetId', 'items'],
-  },
-  annotations: {},
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { testsetId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.testcases.create(testsetId, body)));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/testcases/delete-testcases.ts

@@ -1,56 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'testcases',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'post',
-  httpPath: '/testcases/bulk-delete',
-  operationId: 'deleteTestcases',
-};
-
-export const tool: Tool = {
-  name: 'delete_testcases',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nDelete multiple Testcases by their IDs.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/testcase_delete_response',\n  $defs: {\n    testcase_delete_response: {\n      type: 'object',\n      properties: {\n        success: {\n          type: 'boolean',\n          description: 'Whether the deletion was successful.'\n        }\n      },\n      required: [        'success'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      ids: {
-        type: 'array',
-        description: 'IDs of Testcases to delete.',
-        items: {
-          type: 'string',
-        },
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['ids'],
-  },
-  annotations: {},
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.testcases.delete(body)));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/testcases/get-testcases.ts

@@ -1,54 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'testcases',
-  operation: 'read',
-  tags: [],
-  httpMethod: 'get',
-  httpPath: '/testcases/{testcaseId}',
-  operationId: 'getTestcase',
-};
-
-export const tool: Tool = {
-  name: 'get_testcases',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nRetrieve a specific Testcase by ID.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/testcase',\n  $defs: {\n    testcase: {\n      type: 'object',\n      description: 'A test case in the Scorecard system. Contains JSON data that is validated against the schema defined by its Testset.\\nThe `inputs` and `expected` fields are derived from the `data` field based on the Testset\\'s `fieldMapping`, and include all mapped fields, including those with validation errors.\\nTestcases are stored regardless of validation results, with any validation errors included in the `validationErrors` field.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the Testcase.'\n        },\n        expected: {\n          type: 'object',\n          description: 'Derived from data based on the Testset\\'s fieldMapping. Contains all fields marked as expected outputs, including those with validation errors.',\n          additionalProperties: true\n        },\n        inputs: {\n          type: 'object',\n          description: 'Derived from data based on the Testset\\'s fieldMapping. Contains all fields marked as inputs, including those with validation errors.',\n          additionalProperties: true\n        },\n        jsonData: {\n          type: 'object',\n          description: 'The JSON data of the Testcase, which is validated against the Testset\\'s schema.',\n          additionalProperties: true\n        },\n        testsetId: {\n          type: 'string',\n          description: 'The ID of the Testset this Testcase belongs to.'\n        },\n        validationErrors: {\n          type: 'array',\n          description: 'Validation errors found in the Testcase data. If present, the Testcase doesn\\'t fully conform to its Testset\\'s schema.',\n          items: {\n            type: 'object',\n            properties: {\n              message: {\n                type: 'string',\n                description: 'Human-readable error description.'\n              },\n              path: {\n                type: 'string',\n                description: 'JSON Pointer to the field with the validation error.'\n              }\n            },\n            required: [              'message',\n              'path'\n            ]\n          }\n        }\n      },\n      required: [        'id',\n        'expected',\n        'inputs',\n        'jsonData',\n        'testsetId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      testcaseId: {
-        type: 'string',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['testcaseId'],
-  },
-  annotations: {
-    readOnlyHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { testcaseId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.testcases.get(testcaseId)));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/testcases/list-testcases.ts

@@ -1,65 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'testcases',
-  operation: 'read',
-  tags: [],
-  httpMethod: 'get',
-  httpPath: '/testsets/{testsetId}/testcases',
-  operationId: 'listTestcases',
-};
-
-export const tool: Tool = {
-  name: 'list_testcases',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nRetrieve a paginated list of Testcases belonging to a Testset.\n\n# Response Schema\n```json\n{\n  type: 'object',\n  properties: {\n    data: {\n      type: 'array',\n      items: {\n        $ref: '#/$defs/testcase'\n      }\n    },\n    hasMore: {\n      type: 'boolean'\n    },\n    nextCursor: {\n      type: 'string'\n    },\n    total: {\n      type: 'integer'\n    }\n  },\n  required: [    'data',\n    'hasMore',\n    'nextCursor'\n  ],\n  $defs: {\n    testcase: {\n      type: 'object',\n      description: 'A test case in the Scorecard system. Contains JSON data that is validated against the schema defined by its Testset.\\nThe `inputs` and `expected` fields are derived from the `data` field based on the Testset\\'s `fieldMapping`, and include all mapped fields, including those with validation errors.\\nTestcases are stored regardless of validation results, with any validation errors included in the `validationErrors` field.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the Testcase.'\n        },\n        expected: {\n          type: 'object',\n          description: 'Derived from data based on the Testset\\'s fieldMapping. Contains all fields marked as expected outputs, including those with validation errors.',\n          additionalProperties: true\n        },\n        inputs: {\n          type: 'object',\n          description: 'Derived from data based on the Testset\\'s fieldMapping. Contains all fields marked as inputs, including those with validation errors.',\n          additionalProperties: true\n        },\n        jsonData: {\n          type: 'object',\n          description: 'The JSON data of the Testcase, which is validated against the Testset\\'s schema.',\n          additionalProperties: true\n        },\n        testsetId: {\n          type: 'string',\n          description: 'The ID of the Testset this Testcase belongs to.'\n        },\n        validationErrors: {\n          type: 'array',\n          description: 'Validation errors found in the Testcase data. If present, the Testcase doesn\\'t fully conform to its Testset\\'s schema.',\n          items: {\n            type: 'object',\n            properties: {\n              message: {\n                type: 'string',\n                description: 'Human-readable error description.'\n              },\n              path: {\n                type: 'string',\n                description: 'JSON Pointer to the field with the validation error.'\n              }\n            },\n            required: [              'message',\n              'path'\n            ]\n          }\n        }\n      },\n      required: [        'id',\n        'expected',\n        'inputs',\n        'jsonData',\n        'testsetId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      testsetId: {
-        type: 'string',
-      },
-      cursor: {
-        type: 'string',
-        description:
-          'Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of results.',
-      },
-      limit: {
-        type: 'integer',
-        description:
-          'Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['testsetId'],
-  },
-  annotations: {
-    readOnlyHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { testsetId, jq_filter, ...body } = args as any;
-  const response = await client.testcases.list(testsetId, body).asResponse();
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await response.json()));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/testcases/update-testcases.ts

@@ -1,59 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'testcases',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'put',
-  httpPath: '/testcases/{testcaseId}',
-  operationId: 'updateTestcase',
-};
-
-export const tool: Tool = {
-  name: 'update_testcases',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nReplace the data of an existing Testcase while keeping its ID.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/testcase',\n  $defs: {\n    testcase: {\n      type: 'object',\n      description: 'A test case in the Scorecard system. Contains JSON data that is validated against the schema defined by its Testset.\\nThe `inputs` and `expected` fields are derived from the `data` field based on the Testset\\'s `fieldMapping`, and include all mapped fields, including those with validation errors.\\nTestcases are stored regardless of validation results, with any validation errors included in the `validationErrors` field.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the Testcase.'\n        },\n        expected: {\n          type: 'object',\n          description: 'Derived from data based on the Testset\\'s fieldMapping. Contains all fields marked as expected outputs, including those with validation errors.',\n          additionalProperties: true\n        },\n        inputs: {\n          type: 'object',\n          description: 'Derived from data based on the Testset\\'s fieldMapping. Contains all fields marked as inputs, including those with validation errors.',\n          additionalProperties: true\n        },\n        jsonData: {\n          type: 'object',\n          description: 'The JSON data of the Testcase, which is validated against the Testset\\'s schema.',\n          additionalProperties: true\n        },\n        testsetId: {\n          type: 'string',\n          description: 'The ID of the Testset this Testcase belongs to.'\n        },\n        validationErrors: {\n          type: 'array',\n          description: 'Validation errors found in the Testcase data. If present, the Testcase doesn\\'t fully conform to its Testset\\'s schema.',\n          items: {\n            type: 'object',\n            properties: {\n              message: {\n                type: 'string',\n                description: 'Human-readable error description.'\n              },\n              path: {\n                type: 'string',\n                description: 'JSON Pointer to the field with the validation error.'\n              }\n            },\n            required: [              'message',\n              'path'\n            ]\n          }\n        }\n      },\n      required: [        'id',\n        'expected',\n        'inputs',\n        'jsonData',\n        'testsetId'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      testcaseId: {
-        type: 'string',
-      },
-      jsonData: {
-        type: 'object',
-        description: "The JSON data of the Testcase, which is validated against the Testset's schema.",
-        additionalProperties: true,
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['testcaseId', 'jsonData'],
-  },
-  annotations: {
-    idempotentHint: true,
-  },
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { testcaseId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.testcases.update(testcaseId, body)));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };

package/src/tools/testsets/create-testsets.ts

@@ -1,94 +0,0 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
-
-import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import Scorecard from 'scorecard-ai';
-
-export const metadata: Metadata = {
-  resource: 'testsets',
-  operation: 'write',
-  tags: [],
-  httpMethod: 'post',
-  httpPath: '/projects/{projectId}/testsets',
-  operationId: 'createTestset',
-};
-
-export const tool: Tool = {
-  name: 'create_testsets',
-  description:
-    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nCreate a new Testset for a Project. The Testset will be created in the Project specified in the path.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/testset',\n  $defs: {\n    testset: {\n      type: 'object',\n      description: 'A collection of Testcases that share the same schema.\\nEach Testset defines the structure of its Testcases through a JSON schema.\\nThe `fieldMapping` object maps top-level keys of the Testcase schema to their roles (input/expected output).\\nFields not mentioned in the `fieldMapping` during creation or update are treated as metadata.\\n\\n## JSON Schema validation constraints supported:\\n\\n- **Required fields** - Fields listed in the schema\\'s `required` array must be present in Testcases.\\n- **Type validation** - Values must match the specified type (string, number, boolean, null, integer, object, array).\\n- **Enum validation** - Values must be one of the options specified in the `enum` array.\\n- **Object property validation** - Properties of objects must conform to their defined schemas.\\n- **Array item validation** - Items in arrays must conform to the `items` schema.\\n- **Logical composition** - Values must conform to at least one schema in the `anyOf` array.\\n\\nTestcases that fail validation will still be stored, but will include `validationErrors` detailing the issues.\\nExtra fields in the Testcase data that are not in the schema will be stored but are ignored during validation.',\n      properties: {\n        id: {\n          type: 'string',\n          description: 'The ID of the Testset.'\n        },\n        description: {\n          type: 'string',\n          description: 'The description of the Testset.'\n        },\n        fieldMapping: {\n          type: 'object',\n          description: 'Maps top-level keys of the Testcase schema to their roles (input/expected output). Unmapped fields are treated as metadata.',\n          properties: {\n            expected: {\n              type: 'array',\n              description: 'Fields that represent expected outputs.',\n              items: {\n                type: 'string'\n              }\n            },\n            inputs: {\n              type: 'array',\n              description: 'Fields that represent inputs to the AI system.',\n              items: {\n                type: 'string'\n              }\n            },\n            metadata: {\n              type: 'array',\n              description: 'Fields that are not inputs or expected outputs.',\n              items: {\n                type: 'string'\n              }\n            }\n          },\n          required: [            'expected',\n            'inputs',\n            'metadata'\n          ]\n        },\n        jsonSchema: {\n          type: 'object',\n          description: 'The JSON schema for each Testcase in the Testset.',\n          additionalProperties: true\n        },\n        name: {\n          type: 'string',\n          description: 'The name of the Testset.'\n        }\n      },\n      required: [        'id',\n        'description',\n        'fieldMapping',\n        'jsonSchema',\n        'name'\n      ]\n    }\n  }\n}\n```",
-  inputSchema: {
-    type: 'object',
-    properties: {
-      projectId: {
-        type: 'string',
-      },
-      description: {
-        type: 'string',
-        description: 'The description of the Testset.',
-      },
-      fieldMapping: {
-        type: 'object',
-        description:
-          'Maps top-level keys of the Testcase schema to their roles (input/expected output). Unmapped fields are treated as metadata.',
-        properties: {
-          expected: {
-            type: 'array',
-            description: 'Fields that represent expected outputs.',
-            items: {
-              type: 'string',
-            },
-          },
-          inputs: {
-            type: 'array',
-            description: 'Fields that represent inputs to the AI system.',
-            items: {
-              type: 'string',
-            },
-          },
-          metadata: {
-            type: 'array',
-            description: 'Fields that are not inputs or expected outputs.',
-            items: {
-              type: 'string',
-            },
-          },
-        },
-        required: ['expected', 'inputs', 'metadata'],
-      },
-      jsonSchema: {
-        type: 'object',
-        description: 'The JSON schema for each Testcase in the Testset.',
-        additionalProperties: true,
-      },
-      name: {
-        type: 'string',
-        description: 'The name of the Testset.',
-      },
-      jq_filter: {
-        type: 'string',
-        title: 'jq Filter',
-        description:
-          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
-      },
-    },
-    required: ['projectId', 'description', 'fieldMapping', 'jsonSchema', 'name'],
-  },
-  annotations: {},
-};
-
-export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
-  const { projectId, jq_filter, ...body } = args as any;
-  try {
-    return asTextContentResult(await maybeFilter(jq_filter, await client.testsets.create(projectId, body)));
-  } catch (error) {
-    if (isJqError(error)) {
-      return asErrorResult(error.message);
-    }
-    throw error;
-  }
-};
-
-export default { metadata, tool, handler };