scorecard-ai-mcp 2.4.0 → 2.5.0

package/src/code-tool-worker.ts

@@ -1,11 +1,194 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 import util from 'node:util';
+
+import Fuse from 'fuse.js';
+import ts from 'typescript';
+
 import { WorkerInput, WorkerSuccess, WorkerError } from './code-tool-types';
 import { Scorecard } from 'scorecard-ai';
 
+function getRunFunctionNode(
+  code: string,
+): ts.FunctionDeclaration | ts.FunctionExpression | ts.ArrowFunction | null {
+  const sourceFile = ts.createSourceFile('code.ts', code, ts.ScriptTarget.Latest, true);
+
+  for (const statement of sourceFile.statements) {
+    // Check for top-level function declarations
+    if (ts.isFunctionDeclaration(statement)) {
+      if (statement.name?.text === 'run') {
+        return statement;
+      }
+    }
+
+    // Check for variable declarations: const run = () => {} or const run = function() {}
+    if (ts.isVariableStatement(statement)) {
+      for (const declaration of statement.declarationList.declarations) {
+        if (ts.isIdentifier(declaration.name) && declaration.name.text === 'run') {
+          // Check if it's initialized with a function
+          if (
+            declaration.initializer &&
+            (ts.isFunctionExpression(declaration.initializer) || ts.isArrowFunction(declaration.initializer))
+          ) {
+            return declaration.initializer;
+          }
+        }
+      }
+    }
+  }
+
+  return null;
+}
+
+const fuse = new Fuse(
+  [
+    'client.projects.create',
+    'client.projects.list',
+    'client.testsets.create',
+    'client.testsets.delete',
+    'client.testsets.get',
+    'client.testsets.list',
+    'client.testsets.update',
+    'client.testcases.create',
+    'client.testcases.delete',
+    'client.testcases.get',
+    'client.testcases.list',
+    'client.testcases.update',
+    'client.runs.create',
+    'client.runs.get',
+    'client.runs.list',
+    'client.metrics.create',
+    'client.metrics.delete',
+    'client.metrics.get',
+    'client.metrics.list',
+    'client.metrics.update',
+    'client.records.create',
+    'client.records.delete',
+    'client.records.list',
+    'client.scores.upsert',
+    'client.systems.delete',
+    'client.systems.get',
+    'client.systems.list',
+    'client.systems.update',
+    'client.systems.upsert',
+    'client.systems.versions.get',
+    'client.systems.versions.upsert',
+  ],
+  { threshold: 1, shouldSort: true },
+);
+
+function getMethodSuggestions(fullyQualifiedMethodName: string): string[] {
+  return fuse
+    .search(fullyQualifiedMethodName)
+    .map(({ item }) => item)
+    .slice(0, 5);
+}
+
+const proxyToObj = new WeakMap<any, any>();
+const objToProxy = new WeakMap<any, any>();
+
+type ClientProxyConfig = {
+  path: string[];
+  isBelievedBad?: boolean;
+};
+
+function makeSdkProxy<T extends object>(obj: T, { path, isBelievedBad = false }: ClientProxyConfig): T {
+  let proxy: T = objToProxy.get(obj);
+
+  if (!proxy) {
+    proxy = new Proxy(obj, {
+      get(target, prop, receiver) {
+        const propPath = [...path, String(prop)];
+        const value = Reflect.get(target, prop, receiver);
+
+        if (isBelievedBad || (!(prop in target) && value === undefined)) {
+          // If we're accessing a path that doesn't exist, it will probably eventually error.
+          // Let's proxy it and mark it bad so that we can control the error message.
+          // We proxy an empty class so that an invocation or construction attempt is possible.
+          return makeSdkProxy(class {}, { path: propPath, isBelievedBad: true });
+        }
+
+        if (value !== null && (typeof value === 'object' || typeof value === 'function')) {
+          return makeSdkProxy(value, { path: propPath, isBelievedBad });
+        }
+
+        return value;
+      },
+
+      apply(target, thisArg, args) {
+        if (isBelievedBad || typeof target !== 'function') {
+          const fullyQualifiedMethodName = path.join('.');
+          const suggestions = getMethodSuggestions(fullyQualifiedMethodName);
+          throw new Error(
+            `${fullyQualifiedMethodName} is not a function. Did you mean: ${suggestions.join(', ')}`,
+          );
+        }
+
+        return Reflect.apply(target, proxyToObj.get(thisArg) ?? thisArg, args);
+      },
+
+      construct(target, args, newTarget) {
+        if (isBelievedBad || typeof target !== 'function') {
+          const fullyQualifiedMethodName = path.join('.');
+          const suggestions = getMethodSuggestions(fullyQualifiedMethodName);
+          throw new Error(
+            `${fullyQualifiedMethodName} is not a constructor. Did you mean: ${suggestions.join(', ')}`,
+          );
+        }
+
+        return Reflect.construct(target, args, newTarget);
+      },
+    });
+
+    objToProxy.set(obj, proxy);
+    proxyToObj.set(proxy, obj);
+  }
+
+  return proxy;
+}
+
+function parseError(code: string, error: unknown): string | undefined {
+  if (!(error instanceof Error)) return;
+  const message = error.name ? `${error.name}: ${error.message}` : error.message;
+  try {
+    // Deno uses V8; the first "<anonymous>:LINE:COLUMN" is the top of stack.
+    const lineNumber = error.stack?.match(/<anonymous>:([0-9]+):[0-9]+/)?.[1];
+    // -1 for the zero-based indexing
+    const line =
+      lineNumber &&
+      code
+        .split('\n')
+        .at(parseInt(lineNumber, 10) - 1)
+        ?.trim();
+    return line ? `${message}\n  at line ${lineNumber}\n    ${line}` : message;
+  } catch {
+    return message;
+  }
+}
+
 const fetch = async (req: Request): Promise<Response> => {
   const { opts, code } = (await req.json()) as WorkerInput;
+  if (code == null) {
+    return Response.json(
+      {
+        message:
+          'The code param is missing. Provide one containing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n  // Fill this out\n}\n```',
+      } satisfies WorkerError,
+      { status: 400, statusText: 'Code execution error' },
+    );
+  }
+
+  const runFunctionNode = getRunFunctionNode(code);
+  if (!runFunctionNode) {
+    return Response.json(
+      {
+        message:
+          'The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n  // Fill this out\n}\n```',
+      } satisfies WorkerError,
+      { status: 400, statusText: 'Code execution error' },
+    );
+  }
+
   const client = new Scorecard({
     ...opts,
   });
@@ -22,21 +205,17 @@ const fetch = async (req: Request): Promise<Response> => {
   };
   try {
     let run_ = async (client: any) => {};
-    eval(`
-      ${code}
-      run_ = run;
-    `);
-    const result = await run_(client);
+    eval(`${code}\nrun_ = run;`);
+    const result = await run_(makeSdkProxy(client, { path: ['client'] }));
     return Response.json({
       result,
       logLines,
       errLines,
     } satisfies WorkerSuccess);
   } catch (e) {
-    const message = e instanceof Error ? e.message : undefined;
     return Response.json(
       {
-        message,
+        message: parseError(code, e),
       } satisfies WorkerError,
       { status: 400, statusText: 'Code execution error' },
     );

package/src/code-tool.ts

@@ -3,7 +3,7 @@
 import { dirname } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import Scorecard, { ClientOptions } from 'scorecard-ai';
-import { Endpoint, ContentBlock, Metadata } from './tools/types';
+import { ContentBlock, Endpoint, Metadata, ToolCallResult } from './tools/types';
 
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 
@@ -12,7 +12,7 @@ import { WorkerInput, WorkerError, WorkerSuccess } from './code-tool-types';
 /**
  * A tool that runs code against a copy of the SDK.
  *
- * Instead of exposing every endpoint as it's own tool, which uses up too many tokens for LLMs to use at once,
+ * Instead of exposing every endpoint as its own tool, which uses up too many tokens for LLMs to use at once,
  * we expose a single tool that can be used to search for endpoints by name, resource, operation, or tag, and then
  * a generic endpoint that can be used to invoke any endpoint with the provided arguments.
  *
@@ -23,7 +23,7 @@ export async function codeTool(): Promise<Endpoint> {
   const tool: Tool = {
     name: 'execute',
     description:
-      'Runs Typescript code to interact with the API.\nYou are a skilled programmer writing code to interface with the service.\nDefine an async function named "run" that takes a single parameter of an initialized client, and it will be run.\nDo not initialize a client, but instead use the client that you are given as a parameter.\nYou will be returned anything that your function returns, plus the results of any console.log statements.\nIf any code triggers an error, the tool will return an error response, so you do not need to add error handling unless you want to output something more helpful than the raw error.\nIt is not necessary to add comments to code, unless by adding those comments you believe that you can generate better code.\nThis code will run in a container, and you will not be able to use fetch or otherwise interact with the network calls other than through the client you are given.\nAny variables you define won\'t live between successive uses of this call, so make sure to return or log any data you might need later.',
+      'Runs JavaScript code to interact with the API.\n\nYou are a skilled programmer writing code to interface with the service.\nDefine an async function named "run" that takes a single parameter of an initialized client named "client", and it will be run.\nWrite code within this template:\n\n```\nasync function run(client) {\n  // Fill this out\n}\n```\n\nYou will be returned anything that your function returns, plus the results of any console.log statements.\nIf any code triggers an error, the tool will return an error response, so you do not need to add error handling unless you want to output something more helpful than the raw error.\nIt is not necessary to add comments to code, unless by adding those comments you believe that you can generate better code.\nThis code will run in a container, and you will not be able to use fetch or otherwise interact with the network calls other than through the client you are given.\nAny variables you define won\'t live between successive uses of this call, so make sure to return or log any data you might need later.',
     inputSchema: { type: 'object', properties: { code: { type: 'string' } } },
   };
 
@@ -31,7 +31,7 @@ export async function codeTool(): Promise<Endpoint> {
   const { newDenoHTTPWorker } = await import('@valtown/deno-http-worker');
   const { workerPath } = await import('./code-tool-paths.cjs');
 
-  const handler = async (client: Scorecard, args: unknown) => {
+  const handler = async (client: Scorecard, args: unknown): Promise<ToolCallResult> => {
     const baseURLHostname = new URL(client.baseURL).hostname;
     const { code } = args as { code: string };
 
@@ -97,7 +97,7 @@ export async function codeTool(): Promise<Endpoint> {
         } satisfies WorkerInput);
 
         req.write(body, (err) => {
-          if (err !== null && err !== undefined) {
+          if (err != null) {
             reject(err);
           }
         });
@@ -108,12 +108,12 @@ export async function codeTool(): Promise<Endpoint> {
       if (resp.status === 200) {
         const { result, logLines, errLines } = (await resp.json()) as WorkerSuccess;
         const returnOutput: ContentBlock | null =
-          result === null ? null
-          : result === undefined ? null
-          : {
+          result == null ? null : (
+            {
               type: 'text',
-              text: typeof result === 'string' ? (result as string) : JSON.stringify(result),
-            };
+              text: typeof result === 'string' ? result : JSON.stringify(result),
+            }
+          );
         const logOutput: ContentBlock | null =
           logLines.length === 0 ?
             null
@@ -133,10 +133,11 @@ export async function codeTool(): Promise<Endpoint> {
         };
       } else {
         const { message } = (await resp.json()) as WorkerError;
-        throw new Error(message);
+        return {
+          content: message == null ? [] : [{ type: 'text', text: message }],
+          isError: true,
+        };
       }
-    } catch (e) {
-      throw e;
     } finally {
       worker.terminate();
     }

package/src/dynamic-tools.ts

@@ -14,7 +14,7 @@ function zodToInputSchema(schema: z.ZodSchema) {
 /**
  * A list of tools that expose all the endpoints in the API dynamically.
  *
- * Instead of exposing every endpoint as it's own tool, which uses up too many tokens for LLMs to use at once,
+ * Instead of exposing every endpoint as its own tool, which uses up too many tokens for LLMs to use at once,
  * we expose a single tool that can be used to search for endpoints by name, resource, operation, or tag, and then
  * a generic endpoint that can be used to invoke any endpoint with the provided arguments.
  *

package/src/filtering.ts

@@ -12,3 +12,7 @@ export async function maybeFilter(jqFilter: unknown | undefined, response: any):
 async function jq(json: any, jqFilter: string) {
   return (await initJq).json(json, jqFilter);
 }
+
+export function isJqError(error: any): error is Error {
+  return error instanceof Error && 'stderr' in error;
+}

package/src/headers.ts

@@ -11,7 +11,9 @@ export const parseAuthHeaders = (req: IncomingMessage): Partial<ClientOptions> =
       case 'Bearer':
         return { apiKey: req.headers.authorization.slice('Bearer '.length) };
       default:
-        throw new Error(`Unsupported authorization scheme`);
+        throw new Error(
+          'Unsupported authorization scheme. Expected the "Authorization" header to be a supported scheme (Bearer).',
+        );
     }
   }
 

package/src/http.ts

@@ -46,12 +46,12 @@ const newServer = ({
       },
       mcpOptions,
     });
-  } catch {
+  } catch (error) {
     res.status(401).json({
       jsonrpc: '2.0',
       error: {
         code: -32000,
-        message: 'Unauthorized',
+        message: `Unauthorized: ${error instanceof Error ? error.message : error}`,
       },
     });
     return null;

package/src/options.ts

@@ -284,8 +284,10 @@ const coerceArray = <T extends z.ZodTypeAny>(zodType: T) =>
   );
 
 const QueryOptions = z.object({
-  tools: coerceArray(z.enum(['dynamic', 'all', 'docs'])).describe('Use dynamic tools or all tools'),
-  no_tools: coerceArray(z.enum(['dynamic', 'all', 'docs'])).describe('Do not use dynamic tools or all tools'),
+  tools: coerceArray(z.enum(['dynamic', 'all', 'code', 'docs'])).describe('Specify which MCP tools to use'),
+  no_tools: coerceArray(z.enum(['dynamic', 'all', 'code', 'docs'])).describe(
+    'Specify which MCP tools to not use.',
+  ),
   tool: coerceArray(z.string()).describe('Include tools matching the specified names'),
   resource: coerceArray(z.string()).describe('Include tools matching the specified resources'),
   operation: coerceArray(z.enum(['read', 'write'])).describe(
@@ -385,11 +387,16 @@ export function parseQueryOptions(defaultOptions: McpOptions, query: unknown): M
     : queryOptions.tools?.includes('docs') ? true
     : defaultOptions.includeDocsTools;
 
+  let codeTools: boolean | undefined =
+    queryOptions.no_tools && queryOptions.no_tools?.includes('code') ? false
+    : queryOptions.tools?.includes('code') && defaultOptions.includeCodeTools ? true
+    : defaultOptions.includeCodeTools;
+
   return {
     client: queryOptions.client ?? defaultOptions.client,
     includeDynamicTools: dynamicTools,
     includeAllTools: allTools,
-    includeCodeTools: undefined,
+    includeCodeTools: codeTools,
     includeDocsTools: docsTools,
     filters,
     capabilities: clientCapabilities,

package/src/server.ts

@@ -34,7 +34,7 @@ export const newMcpServer = () =>
   new McpServer(
     {
       name: 'scorecard_ai_api',
-      version: '2.4.0',
+      version: '2.5.0',
     },
     { capabilities: { tools: {}, logging: {} } },
   );

package/src/tools/index.ts

@@ -22,6 +22,7 @@ import get_runs from './runs/get-runs';
 import create_metrics from './metrics/create-metrics';
 import update_metrics from './metrics/update-metrics';
 import list_metrics from './metrics/list-metrics';
+import delete_metrics from './metrics/delete-metrics';
 import get_metrics from './metrics/get-metrics';
 import create_records from './records/create-records';
 import list_records from './records/list-records';
@@ -59,6 +60,7 @@ addEndpoint(get_runs);
 addEndpoint(create_metrics);
 addEndpoint(update_metrics);
 addEndpoint(list_metrics);
+addEndpoint(delete_metrics);
 addEndpoint(get_metrics);
 addEndpoint(create_records);
 addEndpoint(list_records);

package/src/tools/metrics/delete-metrics.ts

@@ -0,0 +1,54 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
+import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+
+import { Tool } from '@modelcontextprotocol/sdk/types.js';
+import Scorecard from 'scorecard-ai';
+
+export const metadata: Metadata = {
+  resource: 'metrics',
+  operation: 'write',
+  tags: [],
+  httpMethod: 'delete',
+  httpPath: '/metrics/{metricId}',
+  operationId: 'deleteMetric',
+};
+
+export const tool: Tool = {
+  name: 'delete_metrics',
+  description:
+    "When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nDelete a specific Metric by ID. The metric will be removed from metric groups and monitors.\n\n# Response Schema\n```json\n{\n  $ref: '#/$defs/metric_delete_response',\n  $defs: {\n    metric_delete_response: {\n      type: 'object',\n      properties: {\n        success: {\n          type: 'boolean',\n          description: 'Whether the deletion was successful.'\n        }\n      },\n      required: [        'success'\n      ]\n    }\n  }\n}\n```",
+  inputSchema: {
+    type: 'object',
+    properties: {
+      metricId: {
+        type: 'string',
+      },
+      jq_filter: {
+        type: 'string',
+        title: 'jq Filter',
+        description:
+          'A jq filter to apply to the response to include certain fields. Consult the output schema in the tool description to see the fields that are available.\n\nFor example: to include only the `name` field in every object of a results array, you can provide ".results[].name".\n\nFor more information, see the [jq documentation](https://jqlang.org/manual/).',
+      },
+    },
+    required: ['metricId'],
+  },
+  annotations: {
+    idempotentHint: true,
+  },
+};
+
+export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
+  const { metricId, jq_filter, ...body } = args as any;
+  try {
+    return asTextContentResult(await maybeFilter(jq_filter, await client.metrics.delete(metricId)));
+  } catch (error) {
+    if (isJqError(error)) {
+      return asErrorResult(error.message);
+    }
+    throw error;
+  }
+};
+
+export default { metadata, tool, handler };

package/src/tools/projects/create-projects.ts

@@ -1,7 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-import { maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
+import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
 
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 import Scorecard from 'scorecard-ai';
@@ -44,7 +44,14 @@ export const tool: Tool = {
 
 export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
   const { jq_filter, ...body } = args as any;
-  return asTextContentResult(await maybeFilter(jq_filter, await client.projects.create(body)));
+  try {
+    return asTextContentResult(await maybeFilter(jq_filter, await client.projects.create(body)));
+  } catch (error) {
+    if (isJqError(error)) {
+      return asErrorResult(error.message);
+    }
+    throw error;
+  }
 };
 
 export default { metadata, tool, handler };

package/src/tools/projects/list-projects.ts

@@ -1,7 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-import { maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
+import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
 
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 import Scorecard from 'scorecard-ai';
@@ -49,7 +49,14 @@ export const tool: Tool = {
 export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
   const { jq_filter, ...body } = args as any;
   const response = await client.projects.list(body).asResponse();
-  return asTextContentResult(await maybeFilter(jq_filter, await response.json()));
+  try {
+    return asTextContentResult(await maybeFilter(jq_filter, await response.json()));
+  } catch (error) {
+    if (isJqError(error)) {
+      return asErrorResult(error.message);
+    }
+    throw error;
+  }
 };
 
 export default { metadata, tool, handler };

package/src/tools/records/create-records.ts

@@ -1,7 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-import { maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
+import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
 
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 import Scorecard from 'scorecard-ai';
@@ -58,7 +58,14 @@ export const tool: Tool = {
 
 export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
   const { runId, jq_filter, ...body } = args as any;
-  return asTextContentResult(await maybeFilter(jq_filter, await client.records.create(runId, body)));
+  try {
+    return asTextContentResult(await maybeFilter(jq_filter, await client.records.create(runId, body)));
+  } catch (error) {
+    if (isJqError(error)) {
+      return asErrorResult(error.message);
+    }
+    throw error;
+  }
 };
 
 export default { metadata, tool, handler };

package/src/tools/records/delete-records.ts

@@ -1,7 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-import { maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
+import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
 
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 import Scorecard from 'scorecard-ai';
@@ -41,7 +41,14 @@ export const tool: Tool = {
 
 export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
   const { recordId, jq_filter, ...body } = args as any;
-  return asTextContentResult(await maybeFilter(jq_filter, await client.records.delete(recordId)));
+  try {
+    return asTextContentResult(await maybeFilter(jq_filter, await client.records.delete(recordId)));
+  } catch (error) {
+    if (isJqError(error)) {
+      return asErrorResult(error.message);
+    }
+    throw error;
+  }
 };
 
 export default { metadata, tool, handler };

package/src/tools/records/list-records.ts

@@ -1,7 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-import { maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
+import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
 
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 import Scorecard from 'scorecard-ai';
@@ -52,7 +52,14 @@ export const tool: Tool = {
 export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
   const { runId, jq_filter, ...body } = args as any;
   const response = await client.records.list(runId, body).asResponse();
-  return asTextContentResult(await maybeFilter(jq_filter, await response.json()));
+  try {
+    return asTextContentResult(await maybeFilter(jq_filter, await response.json()));
+  } catch (error) {
+    if (isJqError(error)) {
+      return asErrorResult(error.message);
+    }
+    throw error;
+  }
 };
 
 export default { metadata, tool, handler };

package/src/tools/runs/create-runs.ts

@@ -1,7 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-import { maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
+import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
 
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 import Scorecard from 'scorecard-ai';
@@ -54,7 +54,14 @@ export const tool: Tool = {
 
 export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
   const { projectId, jq_filter, ...body } = args as any;
-  return asTextContentResult(await maybeFilter(jq_filter, await client.runs.create(projectId, body)));
+  try {
+    return asTextContentResult(await maybeFilter(jq_filter, await client.runs.create(projectId, body)));
+  } catch (error) {
+    if (isJqError(error)) {
+      return asErrorResult(error.message);
+    }
+    throw error;
+  }
 };
 
 export default { metadata, tool, handler };

package/src/tools/runs/get-runs.ts

@@ -1,7 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-import { maybeFilter } from 'scorecard-ai-mcp/filtering';
-import { Metadata, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
+import { isJqError, maybeFilter } from 'scorecard-ai-mcp/filtering';
+import { Metadata, asErrorResult, asTextContentResult } from 'scorecard-ai-mcp/tools/types';
 
 import { Tool } from '@modelcontextprotocol/sdk/types.js';
 import Scorecard from 'scorecard-ai';
@@ -41,7 +41,14 @@ export const tool: Tool = {
 
 export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
   const { runId, jq_filter, ...body } = args as any;
-  return asTextContentResult(await maybeFilter(jq_filter, await client.runs.get(runId)));
+  try {
+    return asTextContentResult(await maybeFilter(jq_filter, await client.runs.get(runId)));
+  } catch (error) {
+    if (isJqError(error)) {
+      return asErrorResult(error.message);
+    }
+    throw error;
+  }
 };
 
 export default { metadata, tool, handler };