scorecard-ai-mcp 1.0.0-alpha.4

package/src/tools/index.ts

@@ -0,0 +1,129 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import Scorecard from 'scorecard-ai';
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+
+import list_projects from './projects/list-projects';
+import create_testsets from './testsets/create-testsets';
+import update_testsets from './testsets/update-testsets';
+import list_testsets from './testsets/list-testsets';
+import delete_testsets from './testsets/delete-testsets';
+import get_testsets from './testsets/get-testsets';
+import create_testcases from './testcases/create-testcases';
+import update_testcases from './testcases/update-testcases';
+import list_testcases from './testcases/list-testcases';
+import delete_testcases from './testcases/delete-testcases';
+import get_testcases from './testcases/get-testcases';
+import create_runs from './runs/create-runs';
+import update_runs from './runs/update-runs';
+import create_records from './records/create-records';
+import create_systems from './systems/create-systems';
+import update_systems from './systems/update-systems';
+import list_systems from './systems/list-systems';
+import delete_systems from './systems/delete-systems';
+import get_systems from './systems/get-systems';
+import create_system_configs from './system-configs/create-system-configs';
+import list_system_configs from './system-configs/list-system-configs';
+import get_system_configs from './system-configs/get-system-configs';
+
+export type HandlerFunction = (client: Scorecard, args: Record<string, unknown> | undefined) => Promise<any>;
+
+export type Metadata = {
+  resource: string;
+  operation: 'read' | 'write';
+  tags: string[];
+};
+
+export type Endpoint = {
+  metadata: Metadata;
+  tool: Tool;
+  handler: HandlerFunction;
+};
+
+export const endpoints: Endpoint[] = [];
+
+function addEndpoint(endpoint: Endpoint) {
+  endpoints.push(endpoint);
+}
+
+addEndpoint(list_projects);
+addEndpoint(create_testsets);
+addEndpoint(update_testsets);
+addEndpoint(list_testsets);
+addEndpoint(delete_testsets);
+addEndpoint(get_testsets);
+addEndpoint(create_testcases);
+addEndpoint(update_testcases);
+addEndpoint(list_testcases);
+addEndpoint(delete_testcases);
+addEndpoint(get_testcases);
+addEndpoint(create_runs);
+addEndpoint(update_runs);
+addEndpoint(create_records);
+addEndpoint(create_systems);
+addEndpoint(update_systems);
+addEndpoint(list_systems);
+addEndpoint(delete_systems);
+addEndpoint(get_systems);
+addEndpoint(create_system_configs);
+addEndpoint(list_system_configs);
+addEndpoint(get_system_configs);
+
+export type Filter = {
+  type: 'resource' | 'operation' | 'tag' | 'tool';
+  op: 'include' | 'exclude';
+  value: string;
+};
+
+export function query(filters: Filter[], endpoints: Endpoint[]): Endpoint[] {
+  if (filters.length === 0) {
+    return endpoints;
+  }
+
+  const allExcludes = filters.every((filter) => filter.op === 'exclude');
+  const unmatchedFilters = new Set(filters);
+
+  const filtered = endpoints.filter((endpoint: Endpoint) => {
+    let included = false || allExcludes;
+
+    for (const filter of filters) {
+      if (match(filter, endpoint)) {
+        unmatchedFilters.delete(filter);
+        included = filter.op === 'include';
+      }
+    }
+
+    return included;
+  });
+
+  // Check if any filters didn't match
+  if (unmatchedFilters.size > 0) {
+    throw new Error(
+      `The following filters did not match any endpoints: ${[...unmatchedFilters]
+        .map((f) => `${f.type}=${f.value}`)
+        .join(', ')}`,
+    );
+  }
+
+  return filtered;
+}
+
+function match({ type, value }: Filter, endpoint: Endpoint): boolean {
+  switch (type) {
+    case 'resource': {
+      const regexStr = '^' + normalizeResource(value).replace(/\*/g, '.*') + '$';
+      const regex = new RegExp(regexStr);
+      return regex.test(normalizeResource(endpoint.metadata.resource));
+    }
+    case 'operation':
+      return endpoint.metadata.operation === value;
+    case 'tag':
+      return endpoint.metadata.tags.includes(value);
+    case 'tool':
+      return endpoint.tool.name === value;
+  }
+}
+
+function normalizeResource(resource: string): string {
+  return resource.toLowerCase().replace(/[^a-z.*\-_]*/g, '');
+}

package/src/tools/projects/list-projects.ts

@@ -0,0 +1,39 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'projects',
+  operation: 'read',
+  tags: [],
+};
+
+export const tool: Tool = {
+  name: 'list_projects',
+  description:
+    'Retrieve a paginated list of all Projects. Projects are ordered by creation date, with oldest Projects first.',
+  inputSchema: {
+    type: 'object',
+    properties: {
+      cursor: {
+        type: 'string',
+        description:
+          'Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of results.',
+      },
+      limit: {
+        type: 'integer',
+        description:
+          'Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.',
+      },
+    },
+  },
+};
+
+export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const body = args as any;
+  return client.projects.list(body);
+};
+
+export default { metadata, tool, handler };

package/src/tools/records/create-records.ts

@@ -0,0 +1,47 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'records',
+  operation: 'write',
+  tags: [],
+};
+
+export const tool: Tool = {
+  name: 'create_records',
+  description: 'Create a new Record in a Run.',
+  inputSchema: {
+    type: 'object',
+    properties: {
+      runId: {
+        type: 'string',
+      },
+      inputs: {
+        type: 'object',
+        description: "The actual inputs sent to the system, which should match the system's input schema.",
+      },
+      labels: {
+        type: 'object',
+        description: 'The expected outputs for the Testcase.',
+      },
+      outputs: {
+        type: 'object',
+        description: 'The actual outputs from the system.',
+      },
+      testcaseId: {
+        type: 'string',
+        description: 'The ID of the Testcase.',
+      },
+    },
+  },
+};
+
+export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { runId, ...body } = args as any;
+  return client.records.create(runId, body);
+};
+
+export default { metadata, tool, handler };

package/src/tools/runs/create-runs.ts

@@ -0,0 +1,46 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'runs',
+  operation: 'write',
+  tags: [],
+};
+
+export const tool: Tool = {
+  name: 'create_runs',
+  description: 'Create a new Run.',
+  inputSchema: {
+    type: 'object',
+    properties: {
+      projectId: {
+        type: 'string',
+      },
+      metricIds: {
+        type: 'array',
+        description: 'The IDs of the metrics this Run is using.',
+        items: {
+          type: 'string',
+        },
+      },
+      testsetId: {
+        type: 'string',
+        description: 'The ID of the Testset this Run is testing.',
+      },
+      systemConfigId: {
+        type: 'string',
+        description: 'The ID of the system configuration this Run is using.',
+      },
+    },
+  },
+};
+
+export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { projectId, ...body } = args as any;
+  return client.runs.create(projectId, body);
+};
+
+export default { metadata, tool, handler };

package/src/tools/runs/update-runs.ts

@@ -0,0 +1,44 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'runs',
+  operation: 'write',
+  tags: [],
+};
+
+export const tool: Tool = {
+  name: 'update_runs',
+  description: 'Update the status of a Run.',
+  inputSchema: {
+    type: 'object',
+    properties: {
+      runId: {
+        type: 'string',
+      },
+      status: {
+        type: 'string',
+        description: 'The status of the Run.',
+        enum: [
+          'pending',
+          'awaiting_execution',
+          'running_execution',
+          'awaiting_scoring',
+          'running_scoring',
+          'awaiting_human_scoring',
+          'completed',
+        ],
+      },
+    },
+  },
+};
+
+export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { runId, ...body } = args as any;
+  return client.runs.update(runId, body);
+};
+
+export default { metadata, tool, handler };

package/src/tools/system-configs/create-system-configs.ts

@@ -0,0 +1,59 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'system_configs',
+  operation: 'write',
+  tags: [],
+};
+
+export const tool: Tool = {
+  name: 'create_system_configs',
+  description:
+    "Create a new configuration for a system.\n\nEach configuration contains specific parameter values that match the system's configSchema - things like model parameters, thresholds, or processing options.\nOnce created, configurations cannot be modified, ensuring stable reference points for evaluations.\n\nWhen creating a configuration:\n- The 'config' object is validated against the parent system's configSchema\n- Configurations with validation errors are still stored, with errors included in the response\n- Validation errors indicate fields that don't match the schema but don't prevent creation\n- Having validation errors may affect how some evaluation metrics are calculated",
+  inputSchema: {
+    type: 'object',
+    properties: {
+      systemId: {
+        type: 'string',
+      },
+      config: {
+        type: 'object',
+        description: 'The configuration of the system.',
+      },
+      name: {
+        type: 'string',
+        description: 'The name of the system configuration.',
+      },
+      validationErrors: {
+        type: 'array',
+        description:
+          "Validation errors found in the configuration. If present, the configuration doesn't fully conform to its system's configSchema.",
+        items: {
+          type: 'object',
+          properties: {
+            message: {
+              type: 'string',
+              description: 'Human-readable error description.',
+            },
+            path: {
+              type: 'string',
+              description: 'JSON Pointer to the field with the validation error.',
+            },
+          },
+          required: ['message', 'path'],
+        },
+      },
+    },
+  },
+};
+
+export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { systemId, ...body } = args as any;
+  return client.systemConfigs.create(systemId, body);
+};
+
+export default { metadata, tool, handler };

package/src/tools/system-configs/get-system-configs.ts

@@ -0,0 +1,34 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'system_configs',
+  operation: 'read',
+  tags: [],
+};
+
+export const tool: Tool = {
+  name: 'get_system_configs',
+  description: 'Retrieve a specific system configuration by ID.',
+  inputSchema: {
+    type: 'object',
+    properties: {
+      systemId: {
+        type: 'string',
+      },
+      systemConfigId: {
+        type: 'string',
+      },
+    },
+  },
+};
+
+export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { systemConfigId, ...body } = args as any;
+  return client.systemConfigs.get(systemConfigId, body);
+};
+
+export default { metadata, tool, handler };

package/src/tools/system-configs/list-system-configs.ts

@@ -0,0 +1,42 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'system_configs',
+  operation: 'read',
+  tags: [],
+};
+
+export const tool: Tool = {
+  name: 'list_system_configs',
+  description:
+    'Retrieve a paginated list of configurations for a specific system.\n\nSystem configurations provide concrete parameter values for a System Under Test, defining exactly how the system should be configured during an evaluation run.',
+  inputSchema: {
+    type: 'object',
+    properties: {
+      systemId: {
+        type: 'string',
+      },
+      cursor: {
+        type: 'string',
+        description:
+          'Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of results.',
+      },
+      limit: {
+        type: 'integer',
+        description:
+          'Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.',
+      },
+    },
+  },
+};
+
+export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { systemId, ...body } = args as any;
+  return client.systemConfigs.list(systemId, body);
+};
+
+export default { metadata, tool, handler };

package/src/tools/systems/create-systems.ts

@@ -0,0 +1,52 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'systems',
+  operation: 'write',
+  tags: [],
+};
+
+export const tool: Tool = {
+  name: 'create_systems',
+  description:
+    'Create a new system definition that specifies the interface contracts for a component you want to evaluate.\n\nA system acts as a template that defines three key contracts through JSON Schemas:\n1. Input Schema: What data your system accepts (e.g., user queries, context documents)\n2. Output Schema: What data your system produces (e.g., responses, confidence scores)\n3. Config Schema: What parameters can be adjusted (e.g., model selection, temperature)\n\nThis separation lets you evaluate any system as a black box, focusing on its interface rather than implementation details.',
+  inputSchema: {
+    type: 'object',
+    properties: {
+      projectId: {
+        type: 'string',
+      },
+      configSchema: {
+        type: 'object',
+        description: "The schema of the system's configuration.",
+      },
+      description: {
+        type: 'string',
+        description: 'The description of the system.',
+      },
+      inputSchema: {
+        type: 'object',
+        description: "The schema of the system's inputs.",
+      },
+      name: {
+        type: 'string',
+        description: 'The name of the system.',
+      },
+      outputSchema: {
+        type: 'object',
+        description: "The schema of the system's outputs.",
+      },
+    },
+  },
+};
+
+export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { projectId, ...body } = args as any;
+  return client.systems.create(projectId, body);
+};
+
+export default { metadata, tool, handler };

package/src/tools/systems/delete-systems.ts

@@ -0,0 +1,31 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'systems',
+  operation: 'write',
+  tags: [],
+};
+
+export const tool: Tool = {
+  name: 'delete_systems',
+  description: 'Delete a system definition by ID. This will not delete associated system configurations.',
+  inputSchema: {
+    type: 'object',
+    properties: {
+      systemId: {
+        type: 'string',
+      },
+    },
+  },
+};
+
+export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { systemId, ...body } = args as any;
+  return client.systems.delete(systemId);
+};
+
+export default { metadata, tool, handler };

package/src/tools/systems/get-systems.ts

@@ -0,0 +1,31 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'systems',
+  operation: 'read',
+  tags: [],
+};
+
+export const tool: Tool = {
+  name: 'get_systems',
+  description: 'Retrieve a specific system by ID.',
+  inputSchema: {
+    type: 'object',
+    properties: {
+      systemId: {
+        type: 'string',
+      },
+    },
+  },
+};
+
+export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { systemId, ...body } = args as any;
+  return client.systems.get(systemId);
+};
+
+export default { metadata, tool, handler };

package/src/tools/systems/list-systems.ts

@@ -0,0 +1,41 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'systems',
+  operation: 'read',
+  tags: [],
+};
+
+export const tool: Tool = {
+  name: 'list_systems',
+  description: 'Retrieve a paginated list of all systems. Systems are ordered by creation date.',
+  inputSchema: {
+    type: 'object',
+    properties: {
+      projectId: {
+        type: 'string',
+      },
+      cursor: {
+        type: 'string',
+        description:
+          'Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of results.',
+      },
+      limit: {
+        type: 'integer',
+        description:
+          'Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.',
+      },
+    },
+  },
+};
+
+export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { projectId, ...body } = args as any;
+  return client.systems.list(projectId, body);
+};
+
+export default { metadata, tool, handler };

package/src/tools/systems/update-systems.ts

@@ -0,0 +1,52 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import type { Metadata } from '../';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'systems',
+  operation: 'write',
+  tags: [],
+};
+
+export const tool: Tool = {
+  name: 'update_systems',
+  description:
+    "Update an existing system definition. Only the fields provided in the request body will be updated.\nIf a field is provided, the new content will replace the existing content.\nIf a field is not provided, the existing content will remain unchanged.\n\nWhen updating schemas:\n- The system will accept your changes regardless of compatibility with existing configurations\n- Schema updates won't invalidate existing evaluations or configurations\n- For significant redesigns, creating a new system definition provides a cleaner separation",
+  inputSchema: {
+    type: 'object',
+    properties: {
+      systemId: {
+        type: 'string',
+      },
+      configSchema: {
+        type: 'object',
+        description: "The schema of the system's configuration.",
+      },
+      description: {
+        type: 'string',
+        description: 'The description of the system.',
+      },
+      inputSchema: {
+        type: 'object',
+        description: "The schema of the system's inputs.",
+      },
+      name: {
+        type: 'string',
+        description: 'The name of the system.',
+      },
+      outputSchema: {
+        type: 'object',
+        description: "The schema of the system's outputs.",
+      },
+    },
+  },
+};
+
+export const handler = (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { systemId, ...body } = args as any;
+  return client.systems.update(systemId, body);
+};
+
+export default { metadata, tool, handler };