scorecard-ai-mcp 1.0.0-alpha.4

package/src/options.test.ts

@@ -0,0 +1,193 @@
+import { parseOptions } from './options';
+import { Filter } from './tools';
+
+// Mock process.argv
+const mockArgv = (args: string[]) => {
+  const originalArgv = process.argv;
+  process.argv = ['node', 'test.js', ...args];
+  return () => {
+    process.argv = originalArgv;
+  };
+};
+
+describe('parseOptions', () => {
+  it('should parse basic filter options', () => {
+    const cleanup = mockArgv([
+      '--tool=test-tool',
+      '--resource=test-resource',
+      '--operation=read',
+      '--tag=test-tag',
+    ]);
+
+    const result = parseOptions();
+
+    expect(result.filters).toEqual([
+      { type: 'tag', op: 'include', value: 'test-tag' },
+      { type: 'resource', op: 'include', value: 'test-resource' },
+      { type: 'tool', op: 'include', value: 'test-tool' },
+      { type: 'operation', op: 'include', value: 'read' },
+    ] as Filter[]);
+
+    // Default client capabilities
+    expect(result.capabilities).toEqual({
+      topLevelUnions: true,
+      validJson: true,
+      refs: true,
+      unions: true,
+      formats: true,
+      toolNameLength: undefined,
+    });
+
+    expect(result.list).toBe(false);
+
+    cleanup();
+  });
+
+  it('should parse exclusion filters', () => {
+    const cleanup = mockArgv([
+      '--no-tool=exclude-tool',
+      '--no-resource=exclude-resource',
+      '--no-operation=write',
+      '--no-tag=exclude-tag',
+    ]);
+
+    const result = parseOptions();
+
+    expect(result.filters).toEqual([
+      { type: 'tag', op: 'exclude', value: 'exclude-tag' },
+      { type: 'resource', op: 'exclude', value: 'exclude-resource' },
+      { type: 'tool', op: 'exclude', value: 'exclude-tool' },
+      { type: 'operation', op: 'exclude', value: 'write' },
+    ] as Filter[]);
+
+    expect(result.capabilities).toEqual({
+      topLevelUnions: true,
+      validJson: true,
+      refs: true,
+      unions: true,
+      formats: true,
+      toolNameLength: undefined,
+    });
+
+    cleanup();
+  });
+
+  it('should parse client presets', () => {
+    const cleanup = mockArgv(['--client=openai-agents']);
+
+    const result = parseOptions();
+
+    expect(result.capabilities).toEqual({
+      topLevelUnions: false,
+      validJson: true,
+      refs: true,
+      unions: true,
+      formats: true,
+      toolNameLength: undefined,
+    });
+
+    cleanup();
+  });
+
+  it('should parse individual capabilities', () => {
+    const cleanup = mockArgv([
+      '--capability=top-level-unions',
+      '--capability=valid-json',
+      '--capability=refs',
+      '--capability=unions',
+      '--capability=tool-name-length=40',
+    ]);
+
+    const result = parseOptions();
+
+    expect(result.capabilities).toEqual({
+      topLevelUnions: true,
+      validJson: true,
+      refs: true,
+      unions: true,
+      formats: true,
+      toolNameLength: 40,
+    });
+
+    cleanup();
+  });
+
+  it('should handle list option', () => {
+    const cleanup = mockArgv(['--list']);
+
+    const result = parseOptions();
+
+    expect(result.list).toBe(true);
+
+    cleanup();
+  });
+
+  it('should handle multiple filters of the same type', () => {
+    const cleanup = mockArgv(['--tool=tool1', '--tool=tool2', '--resource=res1', '--resource=res2']);
+
+    const result = parseOptions();
+
+    expect(result.filters).toEqual([
+      { type: 'resource', op: 'include', value: 'res1' },
+      { type: 'resource', op: 'include', value: 'res2' },
+      { type: 'tool', op: 'include', value: 'tool1' },
+      { type: 'tool', op: 'include', value: 'tool2' },
+    ] as Filter[]);
+
+    cleanup();
+  });
+
+  it('should handle comma-separated values in array options', () => {
+    const cleanup = mockArgv([
+      '--tool=tool1,tool2',
+      '--resource=res1,res2',
+      '--capability=top-level-unions,valid-json,unions',
+    ]);
+
+    const result = parseOptions();
+
+    expect(result.filters).toEqual([
+      { type: 'resource', op: 'include', value: 'res1' },
+      { type: 'resource', op: 'include', value: 'res2' },
+      { type: 'tool', op: 'include', value: 'tool1' },
+      { type: 'tool', op: 'include', value: 'tool2' },
+    ] as Filter[]);
+
+    expect(result.capabilities).toEqual({
+      topLevelUnions: true,
+      validJson: true,
+      refs: true,
+      unions: true,
+      formats: true,
+      toolNameLength: undefined,
+    });
+
+    cleanup();
+  });
+
+  it('should handle invalid tool-name-length format', () => {
+    const cleanup = mockArgv(['--capability=tool-name-length=invalid']);
+
+    // Mock console.error to prevent output during test
+    const originalError = console.error;
+    console.error = jest.fn();
+
+    expect(() => parseOptions()).toThrow();
+
+    console.error = originalError;
+    cleanup();
+  });
+
+  it('should handle unknown capability', () => {
+    const cleanup = mockArgv(['--capability=unknown-capability']);
+
+    // Mock console.error to prevent output during test
+    const originalError = console.error;
+    console.error = jest.fn();
+
+    expect(() => parseOptions()).toThrow();
+
+    console.error = originalError;
+    cleanup();
+  });
+});

package/src/options.ts

@@ -0,0 +1,339 @@
+import yargs from 'yargs';
+import { hideBin } from 'yargs/helpers';
+import { endpoints, Filter } from './tools';
+import { ClientCapabilities } from './compat';
+
+type ClientType = 'openai-agents' | 'claude' | 'claude-code' | 'cursor';
+
+// Client presets for compatibility
+// Note that these could change over time as models get better, so this is
+// a best effort.
+const CLIENT_PRESETS: Record<ClientType, ClientCapabilities> = {
+  'openai-agents': {
+    topLevelUnions: false,
+    validJson: true,
+    refs: true,
+    unions: true,
+    formats: true,
+    toolNameLength: undefined,
+  },
+  claude: {
+    topLevelUnions: true,
+    validJson: false,
+    refs: true,
+    unions: true,
+    formats: true,
+    toolNameLength: undefined,
+  },
+  'claude-code': {
+    topLevelUnions: false,
+    validJson: true,
+    refs: true,
+    unions: true,
+    formats: true,
+    toolNameLength: undefined,
+  },
+  cursor: {
+    topLevelUnions: false,
+    validJson: true,
+    refs: false,
+    unions: false,
+    formats: false,
+    toolNameLength: 50,
+  },
+};
+
+export interface ParsedOptions {
+  filters: Filter[];
+  capabilities: ClientCapabilities;
+  list: boolean;
+}
+
+const CAPABILITY_CHOICES = [
+  'top-level-unions',
+  'valid-json',
+  'refs',
+  'unions',
+  'formats',
+  'tool-name-length',
+] as const;
+
+type Capability = (typeof CAPABILITY_CHOICES)[number];
+
+function parseCapabilityValue(cap: string): { name: Capability; value?: number } {
+  if (cap.startsWith('tool-name-length=')) {
+    const parts = cap.split('=');
+    if (parts.length === 2) {
+      const length = parseInt(parts[1]!, 10);
+      if (!isNaN(length)) {
+        return { name: 'tool-name-length', value: length };
+      }
+      throw new Error(`Invalid tool-name-length value: ${parts[1]}. Expected a number.`);
+    }
+    throw new Error(`Invalid format for tool-name-length. Expected tool-name-length=N.`);
+  }
+  if (!CAPABILITY_CHOICES.includes(cap as Capability)) {
+    throw new Error(`Unknown capability: ${cap}. Valid capabilities are: ${CAPABILITY_CHOICES.join(', ')}`);
+  }
+  return { name: cap as Capability };
+}
+
+export function parseOptions(): ParsedOptions {
+  const opts = yargs(hideBin(process.argv))
+    .option('tool', {
+      type: 'string',
+      array: true,
+      description: 'Include tools matching the specified names',
+    })
+    .option('resource', {
+      type: 'string',
+      array: true,
+      description: 'Include tools matching the specified resources',
+    })
+    .option('operation', {
+      type: 'string',
+      array: true,
+      choices: ['read', 'write'],
+      description: 'Include tools matching the specified operations',
+    })
+    .option('tag', {
+      type: 'string',
+      array: true,
+      description: 'Include tools with the specified tags',
+    })
+    .option('no-tool', {
+      type: 'string',
+      array: true,
+      description: 'Exclude tools matching the specified names',
+    })
+    .option('no-resource', {
+      type: 'string',
+      array: true,
+      description: 'Exclude tools matching the specified resources',
+    })
+    .option('no-operation', {
+      type: 'string',
+      array: true,
+      description: 'Exclude tools matching the specified operations',
+    })
+    .option('no-tag', {
+      type: 'string',
+      array: true,
+      description: 'Exclude tools with the specified tags',
+    })
+    .option('list', {
+      type: 'boolean',
+      description: 'List all tools and exit',
+    })
+    .option('client', {
+      type: 'string',
+      choices: Object.keys(CLIENT_PRESETS),
+      description: 'Specify the MCP client being used',
+    })
+    .option('capability', {
+      type: 'string',
+      array: true,
+      description: 'Specify client capabilities',
+      coerce: (values: string[]) => {
+        return values.flatMap((v) => v.split(','));
+      },
+    })
+    .option('no-capability', {
+      type: 'string',
+      array: true,
+      description: 'Unset client capabilities',
+      choices: CAPABILITY_CHOICES,
+      coerce: (values: string[]) => {
+        return values.flatMap((v) => v.split(','));
+      },
+    })
+    .option('describe-capabilities', {
+      type: 'boolean',
+      description: 'Print detailed explanation of client capabilities and exit',
+    })
+    .help();
+
+  for (const [command, desc] of examples()) {
+    opts.example(command, desc);
+  }
+
+  const argv = opts.parseSync();
+
+  // Handle describe-capabilities flag
+  if (argv.describeCapabilities) {
+    console.log(getCapabilitiesExplanation());
+    process.exit(0);
+  }
+
+  const filters: Filter[] = [];
+
+  // Helper function to support comma-separated values
+  const splitValues = (values: string[] | undefined): string[] => {
+    if (!values) return [];
+    return values.flatMap((v) => v.split(','));
+  };
+
+  for (const tag of splitValues(argv.tag)) {
+    filters.push({ type: 'tag', op: 'include', value: tag });
+  }
+
+  for (const tag of splitValues(argv.noTag)) {
+    filters.push({ type: 'tag', op: 'exclude', value: tag });
+  }
+
+  for (const resource of splitValues(argv.resource)) {
+    filters.push({ type: 'resource', op: 'include', value: resource });
+  }
+
+  for (const resource of splitValues(argv.noResource)) {
+    filters.push({ type: 'resource', op: 'exclude', value: resource });
+  }
+
+  for (const tool of splitValues(argv.tool)) {
+    filters.push({ type: 'tool', op: 'include', value: tool });
+  }
+
+  for (const tool of splitValues(argv.noTool)) {
+    filters.push({ type: 'tool', op: 'exclude', value: tool });
+  }
+
+  for (const operation of splitValues(argv.operation)) {
+    filters.push({ type: 'operation', op: 'include', value: operation });
+  }
+
+  for (const operation of splitValues(argv.noOperation)) {
+    filters.push({ type: 'operation', op: 'exclude', value: operation });
+  }
+
+  // Parse client capabilities
+  const clientCapabilities: ClientCapabilities = {
+    topLevelUnions: true,
+    validJson: true,
+    refs: true,
+    unions: true,
+    formats: true,
+    toolNameLength: undefined,
+  };
+
+  // Apply client preset if specified
+  if (typeof argv.client === 'string') {
+    const clientKey = argv.client as ClientType;
+    if (CLIENT_PRESETS[clientKey]) {
+      Object.assign(clientCapabilities, CLIENT_PRESETS[clientKey]);
+    }
+  }
+
+  // Apply individual capability overrides
+  if (Array.isArray(argv.capability)) {
+    for (const cap of argv.capability) {
+      const parsedCap = parseCapabilityValue(cap);
+      if (parsedCap.name === 'top-level-unions') {
+        clientCapabilities.topLevelUnions = true;
+      } else if (parsedCap.name === 'valid-json') {
+        clientCapabilities.validJson = true;
+      } else if (parsedCap.name === 'refs') {
+        clientCapabilities.refs = true;
+      } else if (parsedCap.name === 'unions') {
+        clientCapabilities.unions = true;
+      } else if (parsedCap.name === 'formats') {
+        clientCapabilities.formats = true;
+      } else if (parsedCap.name === 'tool-name-length') {
+        clientCapabilities.toolNameLength = parsedCap.value;
+      }
+    }
+  }
+
+  // Handle no-capability options to unset capabilities
+  if (Array.isArray(argv.noCapability)) {
+    for (const cap of argv.noCapability) {
+      if (cap === 'top-level-unions') {
+        clientCapabilities.topLevelUnions = false;
+      } else if (cap === 'valid-json') {
+        clientCapabilities.validJson = false;
+      } else if (cap === 'refs') {
+        clientCapabilities.refs = false;
+      } else if (cap === 'unions') {
+        clientCapabilities.unions = false;
+      } else if (cap === 'formats') {
+        clientCapabilities.formats = false;
+      } else if (cap === 'tool-name-length') {
+        clientCapabilities.toolNameLength = undefined;
+      }
+    }
+  }
+
+  return {
+    filters,
+    capabilities: clientCapabilities,
+    list: argv.list || false,
+  };
+}
+
+function getCapabilitiesExplanation(): string {
+  return `
+Client Capabilities Explanation:
+
+Different Language Models (LLMs) and the MCP clients that use them have varying limitations in how they handle tool schemas. Capability flags allow you to inform the MCP server about these limitations.
+
+When a capability flag is set to false, the MCP server will automatically adjust the tool schemas to work around that limitation, ensuring broader compatibility.
+
+Available Capabilities:
+
+# top-level-unions
+Some clients/LLMs do not support JSON schemas with a union type (anyOf) at the root level. If a client lacks this capability, the MCP server splits tools with top-level unions into multiple separate tools, one for each variant in the union.
+
+# refs
+Some clients/LLMs do not support $ref pointers for schema reuse. If a client lacks this capability, the MCP server automatically inlines all references ($defs) directly into the schema. Properties that would cause circular references are removed during this process.
+
+# valid-json
+Some clients/LLMs may incorrectly send arguments as a JSON-encoded string instead of a proper JSON object. If a client *has* this capability, the MCP server will attempt to parse string values as JSON if the initial validation against the schema fails.
+
+# unions
+Some clients/LLMs do not support union types (anyOf) in JSON schemas. If a client lacks this capability, the MCP server removes all anyOf fields and uses only the first variant as the schema.
+
+# formats
+Some clients/LLMs do not support the 'format' keyword in JSON Schema specifications. If a client lacks this capability, the MCP server removes all format fields and appends the format information to the field's description in parentheses.
+
+# tool-name-length=N
+Some clients/LLMs impose a maximum length on tool names. If this capability is set, the MCP server will automatically truncate tool names exceeding the specified length (N), ensuring uniqueness by appending numbers if necessary.
+
+Client Presets (--client):
+Presets like '--client=openai-agents' or '--client=cursor' automatically configure these capabilities based on current known limitations of those clients, simplifying setup.
+
+Current presets:
+${JSON.stringify(CLIENT_PRESETS, null, 2)}
+  `;
+}
+
+function examples(): [string, string][] {
+  const firstEndpoint = endpoints[0]!;
+  const secondEndpoint =
+    endpoints.find((e) => e.metadata.resource !== firstEndpoint.metadata.resource) || endpoints[1];
+  const tag = endpoints.find((e) => e.metadata.tags.length > 0)?.metadata.tags[0];
+  const otherEndpoint = secondEndpoint || firstEndpoint;
+
+  return [
+    [
+      `--tool="${firstEndpoint.tool.name}" ${secondEndpoint ? `--tool="${secondEndpoint.tool.name}"` : ''}`,
+      'Include tools by name',
+    ],
+    [
+      `--resource="${firstEndpoint.metadata.resource}" --operation="read"`,
+      'Filter by resource and operation',
+    ],
+    [
+      `--resource="${otherEndpoint.metadata.resource}*" --no-tool="${otherEndpoint.tool.name}"`,
+      'Use resource wildcards and exclusions',
+    ],
+    [`--client="cursor"`, 'Adjust schemas to be more compatible with Cursor'],
+    [
+      `--capability="top-level-unions" --capability="tool-name-length=40"`,
+      'Specify individual client capabilities',
+    ],
+    [
+      `--client="cursor" --no-capability="tool-name-length"`,
+      'Use cursor client preset but remove tool name length limit',
+    ],
+    ...(tag ? [[`--tag="${tag}"`, 'Filter based on tags'] as [string, string]] : []),
+  ];
+}

package/src/server.ts

@@ -0,0 +1,98 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { endpoints, HandlerFunction } from './tools';
+import { CallToolRequestSchema, ListToolsRequestSchema, Tool } from '@modelcontextprotocol/sdk/types.js';
+import Scorecard from 'scorecard-ai';
+import { ClientCapabilities, defaultClientCapabilities, parseEmbeddedJSON } from './compat';
+export { endpoints } from './tools';
+
+// Create server instance
+export const server = new McpServer(
+  {
+    name: 'scorecard_ai_api',
+    version: '1.0.0-alpha.4',
+  },
+  {
+    capabilities: {
+      tools: {},
+    },
+  },
+);
+
+/**
+ * Initializes the provided MCP Server with the given tools and handlers.
+ * If not provided, the default client, tools and handlers will be used.
+ */
+export function init(params: {
+  server: Server | McpServer;
+  client?: Scorecard;
+  endpoints?: { tool: Tool; handler: HandlerFunction }[];
+  capabilities?: Partial<ClientCapabilities>;
+}) {
+  const server = params.server instanceof McpServer ? params.server.server : params.server;
+  const providedEndpoints = params.endpoints || endpoints;
+
+  const endpointMap = Object.fromEntries(providedEndpoints.map((endpoint) => [endpoint.tool.name, endpoint]));
+
+  const client = params.client || new Scorecard({});
+
+  server.setRequestHandler(ListToolsRequestSchema, async () => {
+    return {
+      tools: providedEndpoints.map((endpoint) => endpoint.tool),
+    };
+  });
+
+  server.setRequestHandler(CallToolRequestSchema, async (request) => {
+    const { name, arguments: args } = request.params;
+    const endpoint = endpointMap[name];
+    if (!endpoint) {
+      throw new Error(`Unknown tool: ${name}`);
+    }
+
+    return executeHandler(endpoint.tool, endpoint.handler, client, args, params.capabilities);
+  });
+}
+
+/**
+ * Runs the provided handler with the given client and arguments.
+ */
+export async function executeHandler(
+  tool: Tool,
+  handler: HandlerFunction,
+  client: Scorecard,
+  args: Record<string, unknown> | undefined,
+  compatibilityOptions?: Partial<ClientCapabilities>,
+) {
+  const options = { ...defaultClientCapabilities, ...compatibilityOptions };
+  if (options.validJson && args) {
+    args = args = parseEmbeddedJSON(args, tool.inputSchema);
+  }
+  const result = await handler(client, args || {});
+  return {
+    content: [
+      {
+        type: 'text',
+        text: JSON.stringify(result, null, 2),
+      },
+    ],
+  };
+}
+
+export const readEnv = (env: string): string | undefined => {
+  if (typeof (globalThis as any).process !== 'undefined') {
+    return (globalThis as any).process.env?.[env]?.trim();
+  } else if (typeof (globalThis as any).Deno !== 'undefined') {
+    return (globalThis as any).Deno.env?.get?.(env)?.trim();
+  }
+  return;
+};
+
+export const readEnvOrError = (env: string): string => {
+  let envValue = readEnv(env);
+  if (envValue === undefined) {
+    throw new Error(`Environment variable ${env} is not set`);
+  }
+  return envValue;
+};