scorecard-ai-mcp 2.0.0 → 2.1.1

package/src/index.ts

@@ -1,9 +1,10 @@
 #!/usr/bin/env node
 
-import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
-import { init, selectTools, server } from './server';
+import { selectTools } from './server';
 import { Endpoint, endpoints } from './tools';
-import { McpOptions, parseOptions } from './options';
+import { McpOptions, parseCLIOptions } from './options';
+import { launchStdioServer } from './stdio';
+import { launchStreamableHTTPServer } from './http';
 
 async function main() {
   const options = parseOptionsOrError();
@@ -13,18 +14,21 @@ async function main() {
     return;
   }
 
-  const includedTools = selectToolsOrError(endpoints, options);
+  const selectedTools = await selectToolsOrError(endpoints, options);
 
   console.error(
-    `MCP Server starting with ${includedTools.length} tools:`,
-    includedTools.map((e) => e.tool.name),
+    `MCP Server starting with ${selectedTools.length} tools:`,
+    selectedTools.map((e) => e.tool.name),
   );
 
-  init({ server, endpoints: includedTools });
-
-  const transport = new StdioServerTransport();
-  await server.connect(transport);
-  console.error('MCP Server running on stdio');
+  switch (options.transport) {
+    case 'stdio':
+      await launchStdioServer(options);
+      break;
+    case 'http':
+      await launchStreamableHTTPServer(options, options.port ?? options.socket);
+      break;
+  }
 }
 
 if (require.main === module) {
@@ -36,16 +40,16 @@ if (require.main === module) {
 
 function parseOptionsOrError() {
   try {
-    return parseOptions();
+    return parseCLIOptions();
   } catch (error) {
     console.error('Error parsing options:', error);
     process.exit(1);
   }
 }
 
-function selectToolsOrError(endpoints: Endpoint[], options: McpOptions) {
+async function selectToolsOrError(endpoints: Endpoint[], options: McpOptions): Promise<Endpoint[]> {
   try {
-    const includedTools = selectTools(endpoints, options);
+    const includedTools = await selectTools(endpoints, options);
     if (includedTools.length === 0) {
       console.error('No tools match the provided filters.');
       process.exit(1);

package/src/options.ts

@@ -1,18 +1,24 @@
+import qs from 'qs';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
+import z from 'zod';
 import { endpoints, Filter } from './tools';
 import { ClientCapabilities, knownClients, ClientType } from './compat';
 
 export type CLIOptions = McpOptions & {
   list: boolean;
+  transport: 'stdio' | 'http';
+  port: number | undefined;
+  socket: string | undefined;
 };
 
 export type McpOptions = {
-  client: ClientType | undefined;
-  includeDynamicTools: boolean | undefined;
-  includeAllTools: boolean | undefined;
-  filters: Filter[];
-  capabilities?: Partial<ClientCapabilities>;
+  client?: ClientType | undefined;
+  includeDynamicTools?: boolean | undefined;
+  includeAllTools?: boolean | undefined;
+  includeCodeTools?: boolean | undefined;
+  filters?: Filter[] | undefined;
+  capabilities?: Partial<ClientCapabilities> | undefined;
 };
 
 const CAPABILITY_CHOICES = [
@@ -44,18 +50,18 @@ function parseCapabilityValue(cap: string): { name: Capability; value?: number }
   return { name: cap as Capability };
 }
 
-export function parseOptions(): CLIOptions {
+export function parseCLIOptions(): CLIOptions {
   const opts = yargs(hideBin(process.argv))
     .option('tools', {
       type: 'string',
       array: true,
-      choices: ['dynamic', 'all'],
+      choices: ['dynamic', 'all', 'code'],
       description: 'Use dynamic tools or all tools',
     })
     .option('no-tools', {
       type: 'string',
       array: true,
-      choices: ['dynamic', 'all'],
+      choices: ['dynamic', 'all', 'code'],
       description: 'Do not use any dynamic or all tools',
     })
     .option('tool', {
@@ -129,6 +135,20 @@ export function parseOptions(): CLIOptions {
       type: 'boolean',
       description: 'Print detailed explanation of client capabilities and exit',
     })
+    .option('transport', {
+      type: 'string',
+      choices: ['stdio', 'http'],
+      default: 'stdio',
+      description: 'What transport to use; stdio for local servers or http for remote servers',
+    })
+    .option('port', {
+      type: 'number',
+      description: 'Port to serve on if using http transport',
+    })
+    .option('socket', {
+      type: 'string',
+      description: 'Unix socket to serve on if using http transport',
+    })
     .help();
 
   for (const [command, desc] of examples()) {
@@ -184,14 +204,7 @@ export function parseOptions(): CLIOptions {
   }
 
   // Parse client capabilities
-  const clientCapabilities: ClientCapabilities = {
-    topLevelUnions: true,
-    validJson: true,
-    refs: true,
-    unions: true,
-    formats: true,
-    toolNameLength: undefined,
-  };
+  const clientCapabilities: Partial<ClientCapabilities> = {};
 
   // Apply individual capability overrides
   if (Array.isArray(argv.capability)) {
@@ -232,20 +245,144 @@ export function parseOptions(): CLIOptions {
     }
   }
 
+  const shouldIncludeToolType = (toolType: 'dynamic' | 'all' | 'code') =>
+    explicitTools ? argv.tools?.includes(toolType) && !argv.noTools?.includes(toolType) : undefined;
+
   const explicitTools = Boolean(argv.tools || argv.noTools);
-  const includeDynamicTools =
-    explicitTools ? argv.tools?.includes('dynamic') && !argv.noTools?.includes('dynamic') : undefined;
-  const includeAllTools =
-    explicitTools ? argv.tools?.includes('all') && !argv.noTools?.includes('all') : undefined;
+  const includeDynamicTools = shouldIncludeToolType('dynamic');
+  const includeAllTools = shouldIncludeToolType('all');
+  const includeCodeTools = shouldIncludeToolType('code');
+
+  const transport = argv.transport as 'stdio' | 'http';
 
   const client = argv.client as ClientType;
   return {
-    client: client && knownClients[client] ? client : undefined,
+    client: client && client !== 'infer' && knownClients[client] ? client : undefined,
     includeDynamicTools,
     includeAllTools,
+    includeCodeTools,
     filters,
     capabilities: clientCapabilities,
     list: argv.list || false,
+    transport,
+    port: argv.port,
+    socket: argv.socket,
+  };
+}
+
+const coerceArray = <T extends z.ZodTypeAny>(zodType: T) =>
+  z.preprocess(
+    (val) =>
+      Array.isArray(val) ? val
+      : val ? [val]
+      : val,
+    z.array(zodType).optional(),
+  );
+
+const QueryOptions = z.object({
+  tools: coerceArray(z.enum(['dynamic', 'all'])).describe('Use dynamic tools or all tools'),
+  no_tools: coerceArray(z.enum(['dynamic', 'all'])).describe('Do not use dynamic tools or all tools'),
+  tool: coerceArray(z.string()).describe('Include tools matching the specified names'),
+  resource: coerceArray(z.string()).describe('Include tools matching the specified resources'),
+  operation: coerceArray(z.enum(['read', 'write'])).describe(
+    'Include tools matching the specified operations',
+  ),
+  tag: coerceArray(z.string()).describe('Include tools with the specified tags'),
+  no_tool: coerceArray(z.string()).describe('Exclude tools matching the specified names'),
+  no_resource: coerceArray(z.string()).describe('Exclude tools matching the specified resources'),
+  no_operation: coerceArray(z.enum(['read', 'write'])).describe(
+    'Exclude tools matching the specified operations',
+  ),
+  no_tag: coerceArray(z.string()).describe('Exclude tools with the specified tags'),
+  client: ClientType.optional().describe('Specify the MCP client being used'),
+  capability: coerceArray(z.string()).describe('Specify client capabilities'),
+  no_capability: coerceArray(z.enum(CAPABILITY_CHOICES)).describe('Unset client capabilities'),
+});
+
+export function parseQueryOptions(defaultOptions: McpOptions, query: unknown): McpOptions {
+  const queryObject = typeof query === 'string' ? qs.parse(query) : query;
+  const queryOptions = QueryOptions.parse(queryObject);
+
+  const filters: Filter[] = [...(defaultOptions.filters ?? [])];
+
+  for (const resource of queryOptions.resource || []) {
+    filters.push({ type: 'resource', op: 'include', value: resource });
+  }
+  for (const operation of queryOptions.operation || []) {
+    filters.push({ type: 'operation', op: 'include', value: operation });
+  }
+  for (const tag of queryOptions.tag || []) {
+    filters.push({ type: 'tag', op: 'include', value: tag });
+  }
+  for (const tool of queryOptions.tool || []) {
+    filters.push({ type: 'tool', op: 'include', value: tool });
+  }
+  for (const resource of queryOptions.no_resource || []) {
+    filters.push({ type: 'resource', op: 'exclude', value: resource });
+  }
+  for (const operation of queryOptions.no_operation || []) {
+    filters.push({ type: 'operation', op: 'exclude', value: operation });
+  }
+  for (const tag of queryOptions.no_tag || []) {
+    filters.push({ type: 'tag', op: 'exclude', value: tag });
+  }
+  for (const tool of queryOptions.no_tool || []) {
+    filters.push({ type: 'tool', op: 'exclude', value: tool });
+  }
+
+  // Parse client capabilities
+  const clientCapabilities: Partial<ClientCapabilities> = { ...defaultOptions.capabilities };
+
+  for (const cap of queryOptions.capability || []) {
+    const parsed = parseCapabilityValue(cap);
+    if (parsed.name === 'top-level-unions') {
+      clientCapabilities.topLevelUnions = true;
+    } else if (parsed.name === 'valid-json') {
+      clientCapabilities.validJson = true;
+    } else if (parsed.name === 'refs') {
+      clientCapabilities.refs = true;
+    } else if (parsed.name === 'unions') {
+      clientCapabilities.unions = true;
+    } else if (parsed.name === 'formats') {
+      clientCapabilities.formats = true;
+    } else if (parsed.name === 'tool-name-length') {
+      clientCapabilities.toolNameLength = parsed.value;
+    }
+  }
+
+  for (const cap of queryOptions.no_capability || []) {
+    if (cap === 'top-level-unions') {
+      clientCapabilities.topLevelUnions = false;
+    } else if (cap === 'valid-json') {
+      clientCapabilities.validJson = false;
+    } else if (cap === 'refs') {
+      clientCapabilities.refs = false;
+    } else if (cap === 'unions') {
+      clientCapabilities.unions = false;
+    } else if (cap === 'formats') {
+      clientCapabilities.formats = false;
+    } else if (cap === 'tool-name-length') {
+      clientCapabilities.toolNameLength = undefined;
+    }
+  }
+
+  let dynamicTools: boolean | undefined =
+    queryOptions.no_tools && queryOptions.no_tools?.includes('dynamic') ? false
+    : queryOptions.tools?.includes('dynamic') ? true
+    : defaultOptions.includeDynamicTools;
+
+  let allTools: boolean | undefined =
+    queryOptions.no_tools && queryOptions.no_tools?.includes('all') ? false
+    : queryOptions.tools?.includes('all') ? true
+    : defaultOptions.includeAllTools;
+
+  return {
+    client: queryOptions.client ?? defaultOptions.client,
+    includeDynamicTools: dynamicTools,
+    includeAllTools: allTools,
+    includeCodeTools: undefined,
+    filters,
+    capabilities: clientCapabilities,
   };
 }
 

package/src/server.ts

@@ -3,7 +3,13 @@
 import { Server } from '@modelcontextprotocol/sdk/server/index.js';
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { Endpoint, endpoints, HandlerFunction, query } from './tools';
-import { CallToolRequestSchema, ListToolsRequestSchema, Tool } from '@modelcontextprotocol/sdk/types.js';
+import {
+  CallToolRequestSchema,
+  ListToolsRequestSchema,
+  SetLevelRequestSchema,
+  Implementation,
+  Tool,
+} from '@modelcontextprotocol/sdk/types.js';
 import { ClientOptions } from 'scorecard-ai';
 import Scorecard from 'scorecard-ai';
 import {
@@ -14,6 +20,7 @@ import {
   parseEmbeddedJSON,
 } from './compat';
 import { dynamicTools } from './dynamic-tools';
+import { codeTool } from './code-tool';
 import { McpOptions } from './options';
 
 export { McpOptions } from './options';
@@ -22,18 +29,17 @@ export { Filter } from './tools';
 export { ClientOptions } from 'scorecard-ai';
 export { endpoints } from './tools';
 
-// Create server instance
-export const server = new McpServer(
-  {
-    name: 'scorecard_ai_api',
-    version: '2.0.0',
-  },
-  {
-    capabilities: {
-      tools: {},
+export const newMcpServer = () =>
+  new McpServer(
+    {
+      name: 'scorecard_ai_api',
+      version: '2.1.1',
     },
-  },
-);
+    { capabilities: { tools: {}, logging: {} } },
+  );
+
+// Create server instance
+export const server = newMcpServer();
 
 /**
  * Initializes the provided MCP Server with the given tools and handlers.
@@ -41,77 +47,126 @@ export const server = new McpServer(
  */
 export function initMcpServer(params: {
   server: Server | McpServer;
-  clientOptions: ClientOptions;
-  mcpOptions: McpOptions;
-  endpoints?: { tool: Tool; handler: HandlerFunction }[];
-}) {
-  const transformedEndpoints = selectTools(endpoints, params.mcpOptions);
-  const client = new Scorecard(params.clientOptions);
-  const capabilities = {
-    ...defaultClientCapabilities,
-    ...(params.mcpOptions.client ? knownClients[params.mcpOptions.client] : params.mcpOptions.capabilities),
-  };
-  init({ server: params.server, client, endpoints: transformedEndpoints, capabilities });
-}
-
-export function init(params: {
-  server: Server | McpServer;
-  client?: Scorecard;
-  endpoints?: { tool: Tool; handler: HandlerFunction }[];
-  capabilities?: Partial<ClientCapabilities>;
+  clientOptions?: ClientOptions;
+  mcpOptions?: McpOptions;
 }) {
   const server = params.server instanceof McpServer ? params.server.server : params.server;
-  const providedEndpoints = params.endpoints || endpoints;
+  const mcpOptions = params.mcpOptions ?? {};
+
+  let providedEndpoints: Endpoint[] | null = null;
+  let endpointMap: Record<string, Endpoint> | null = null;
+
+  const initTools = async (implementation?: Implementation) => {
+    if (implementation && (!mcpOptions.client || mcpOptions.client === 'infer')) {
+      mcpOptions.client =
+        implementation.name.toLowerCase().includes('claude') ? 'claude'
+        : implementation.name.toLowerCase().includes('cursor') ? 'cursor'
+        : undefined;
+      mcpOptions.capabilities = {
+        ...(mcpOptions.client && knownClients[mcpOptions.client]),
+        ...mcpOptions.capabilities,
+      };
+    }
+    providedEndpoints ??= await selectTools(endpoints, mcpOptions);
+    endpointMap ??= Object.fromEntries(providedEndpoints.map((endpoint) => [endpoint.tool.name, endpoint]));
+  };
 
-  const endpointMap = Object.fromEntries(providedEndpoints.map((endpoint) => [endpoint.tool.name, endpoint]));
+  const logAtLevel =
+    (level: 'debug' | 'info' | 'warning' | 'error') =>
+    (message: string, ...rest: unknown[]) => {
+      void server.sendLoggingMessage({
+        level,
+        data: { message, rest },
+      });
+    };
+  const logger = {
+    debug: logAtLevel('debug'),
+    info: logAtLevel('info'),
+    warn: logAtLevel('warning'),
+    error: logAtLevel('error'),
+  };
 
-  const client =
-    params.client ||
-    new Scorecard({
-      environment: (readEnv('SCORECARD_ENVIRONMENT') || undefined) as any,
-      defaultHeaders: { 'X-Stainless-MCP': 'true' },
-    });
+  let client = new Scorecard({
+    ...{ environment: (readEnv('SCORECARD_ENVIRONMENT') || undefined) as any },
+    logger,
+    ...params.clientOptions,
+    defaultHeaders: {
+      ...params.clientOptions?.defaultHeaders,
+      'X-Stainless-MCP': 'true',
+    },
+  });
 
   server.setRequestHandler(ListToolsRequestSchema, async () => {
+    if (providedEndpoints === null) {
+      await initTools(server.getClientVersion());
+    }
     return {
-      tools: providedEndpoints.map((endpoint) => endpoint.tool),
+      tools: providedEndpoints!.map((endpoint) => endpoint.tool),
     };
   });
 
   server.setRequestHandler(CallToolRequestSchema, async (request) => {
+    if (endpointMap === null) {
+      await initTools(server.getClientVersion());
+    }
     const { name, arguments: args } = request.params;
-    const endpoint = endpointMap[name];
+    const endpoint = endpointMap![name];
     if (!endpoint) {
       throw new Error(`Unknown tool: ${name}`);
     }
 
-    return executeHandler(endpoint.tool, endpoint.handler, client, args, params.capabilities);
+    return executeHandler(endpoint.tool, endpoint.handler, client, args, mcpOptions.capabilities);
+  });
+
+  server.setRequestHandler(SetLevelRequestSchema, async (request) => {
+    const { level } = request.params;
+    switch (level) {
+      case 'debug':
+        client = client.withOptions({ logLevel: 'debug' });
+        break;
+      case 'info':
+        client = client.withOptions({ logLevel: 'info' });
+        break;
+      case 'notice':
+      case 'warning':
+        client = client.withOptions({ logLevel: 'warn' });
+        break;
+      case 'error':
+        client = client.withOptions({ logLevel: 'error' });
+        break;
+      default:
+        client = client.withOptions({ logLevel: 'off' });
+        break;
+    }
+    return {};
   });
 }
 
 /**
  * Selects the tools to include in the MCP Server based on the provided options.
  */
-export function selectTools(endpoints: Endpoint[], options: McpOptions) {
-  const filteredEndpoints = query(options.filters, endpoints);
+export async function selectTools(endpoints: Endpoint[], options?: McpOptions): Promise<Endpoint[]> {
+  const filteredEndpoints = query(options?.filters ?? [], endpoints);
 
   let includedTools = filteredEndpoints;
 
   if (includedTools.length > 0) {
-    if (options.includeDynamicTools) {
+    if (options?.includeDynamicTools) {
       includedTools = dynamicTools(includedTools);
     }
   } else {
-    if (options.includeAllTools) {
+    if (options?.includeAllTools) {
       includedTools = endpoints;
-    } else if (options.includeDynamicTools) {
+    } else if (options?.includeDynamicTools) {
       includedTools = dynamicTools(endpoints);
+    } else if (options?.includeCodeTools) {
+      includedTools = [await codeTool()];
     } else {
       includedTools = endpoints;
     }
   }
 
-  const capabilities = { ...defaultClientCapabilities, ...options.capabilities };
+  const capabilities = { ...defaultClientCapabilities, ...options?.capabilities };
   return applyCompatibilityTransformations(includedTools, capabilities);
 }
 
@@ -126,7 +181,7 @@ export async function executeHandler(
   compatibilityOptions?: Partial<ClientCapabilities>,
 ) {
   const options = { ...defaultClientCapabilities, ...compatibilityOptions };
-  if (options.validJson && args) {
+  if (!options.validJson && args) {
     args = parseEmbeddedJSON(args, tool.inputSchema);
   }
   return await handler(client, args || {});

package/src/stdio.ts

@@ -0,0 +1,13 @@
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { initMcpServer, newMcpServer } from './server';
+import { McpOptions } from './options';
+
+export const launchStdioServer = async (options: McpOptions) => {
+  const server = newMcpServer();
+
+  initMcpServer({ server, mcpOptions: options });
+
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+  console.error('MCP Server running on stdio');
+};

package/src/tools/metrics/create-metrics.ts

@@ -1,9 +1,8 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+import { Metadata, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
 
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import type { Metadata } from '../';
 import Scorecard from 'scorecard-ai';
 
 export const metadata: Metadata = {
@@ -68,6 +67,7 @@ export const tool: Tool = {
             description: 'The temperature for AI evaluation (0-2).',
           },
         },
+        required: ['projectId', 'evalType', 'name', 'outputType', 'promptTemplate'],
       },
       {
         type: 'object',
@@ -102,6 +102,7 @@ export const tool: Tool = {
             description: 'The threshold for determining pass/fail from integer scores (1-5).',
           },
         },
+        required: ['projectId', 'evalType', 'name', 'outputType'],
       },
       {
         type: 'object',
@@ -136,6 +137,7 @@ export const tool: Tool = {
             description: 'The threshold for determining pass/fail from integer scores (1-5).',
           },
         },
+        required: ['projectId', 'evalType', 'name', 'outputType'],
       },
       {
         type: 'object',
@@ -179,6 +181,7 @@ export const tool: Tool = {
             description: 'The temperature for AI evaluation (0-2).',
           },
         },
+        required: ['projectId', 'evalType', 'name', 'outputType', 'promptTemplate'],
       },
       {
         type: 'object',
@@ -209,6 +212,7 @@ export const tool: Tool = {
             description: 'Guidelines for human evaluators.',
           },
         },
+        required: ['projectId', 'evalType', 'name', 'outputType'],
       },
       {
         type: 'object',
@@ -239,9 +243,11 @@ export const tool: Tool = {
             description: 'Optional guidelines for heuristic evaluation logic.',
           },
         },
+        required: ['projectId', 'evalType', 'name', 'outputType'],
       },
     ],
   },
+  annotations: {},
 };
 
 export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {

package/src/tools/metrics/update-metrics.ts

@@ -1,9 +1,8 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+import { Metadata, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
 
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
-import type { Metadata } from '../';
 import Scorecard from 'scorecard-ai';
 
 export const metadata: Metadata = {
@@ -68,6 +67,7 @@ export const tool: Tool = {
             description: 'The temperature for AI evaluation (0-2).',
           },
         },
+        required: ['metricId', 'evalType', 'outputType'],
       },
       {
         type: 'object',
@@ -102,6 +102,7 @@ export const tool: Tool = {
             description: 'The threshold for determining pass/fail from integer scores (1-5).',
           },
         },
+        required: ['metricId', 'evalType', 'outputType'],
       },
       {
         type: 'object',
@@ -136,6 +137,7 @@ export const tool: Tool = {
             description: 'The threshold for determining pass/fail from integer scores (1-5).',
           },
         },
+        required: ['metricId', 'evalType', 'outputType'],
       },
       {
         type: 'object',
@@ -179,6 +181,7 @@ export const tool: Tool = {
             description: 'The temperature for AI evaluation (0-2).',
           },
         },
+        required: ['metricId', 'evalType', 'outputType'],
       },
       {
         type: 'object',
@@ -209,6 +212,7 @@ export const tool: Tool = {
             description: 'The name of the Metric.',
           },
         },
+        required: ['metricId', 'evalType', 'outputType'],
       },
       {
         type: 'object',
@@ -239,9 +243,11 @@ export const tool: Tool = {
             description: 'The name of the Metric.',
           },
         },
+        required: ['metricId', 'evalType', 'outputType'],
       },
     ],
   },
+  annotations: {},
 };
 
 export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {