proctor-mcp-server 0.1.0

package/shared/tools/get-machines.js

@@ -0,0 +1,83 @@
+export function getMachines(_server, clientFactory) {
+    return {
+        name: 'get_machines',
+        description: `List active Fly.io machines used for Proctor exams.
+
+Returns information about currently running or recently active Fly machines
+that are being used for exam execution.
+
+**Returns:**
+- machines: Array of machine objects with id, state, region, and other metadata
+
+**Use cases:**
+- Monitor active exam execution infrastructure
+- Find machines to clean up or cancel
+- Debug issues with running exams
+- Check resource utilization
+
+**Note:**
+- Machines may be in various states (running, stopped, etc.)
+- Use destroy_machine to remove machines that are no longer needed
+- Use cancel_exam to stop a running exam on a specific machine`,
+        inputSchema: {
+            type: 'object',
+            properties: {},
+            required: [],
+        },
+        handler: async () => {
+            const client = clientFactory();
+            try {
+                const response = await client.getMachines();
+                if (response.machines.length === 0) {
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: 'No active Fly.io machines found.',
+                            },
+                        ],
+                    };
+                }
+                let content = `## Active Machines (${response.machines.length})\n\n`;
+                for (const machine of response.machines) {
+                    content += `### Machine: ${machine.id}\n`;
+                    if (machine.name)
+                        content += `- **Name:** ${machine.name}\n`;
+                    if (machine.state)
+                        content += `- **State:** ${machine.state}\n`;
+                    if (machine.region)
+                        content += `- **Region:** ${machine.region}\n`;
+                    if (machine.created_at)
+                        content += `- **Created:** ${machine.created_at}\n`;
+                    // Include any other properties
+                    const otherProps = Object.entries(machine).filter(([key]) => !['id', 'name', 'state', 'region', 'created_at'].includes(key));
+                    if (otherProps.length > 0) {
+                        for (const [key, value] of otherProps) {
+                            content += `- **${key}:** ${JSON.stringify(value)}\n`;
+                        }
+                    }
+                    content += '\n';
+                }
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: content.trim(),
+                        },
+                    ],
+                };
+            }
+            catch (error) {
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: `Error getting machines: ${error instanceof Error ? error.message : String(error)}`,
+                        },
+                    ],
+                    isError: true,
+                };
+            }
+        },
+    };
+}

package/shared/tools/get-metadata.d.ts

@@ -0,0 +1,25 @@
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import type { ClientFactory } from '../server.js';
+export declare function getMetadata(_server: Server, clientFactory: ClientFactory): {
+    name: string;
+    description: string;
+    inputSchema: {
+        type: string;
+        properties: {};
+        required: never[];
+    };
+    handler: () => Promise<{
+        content: {
+            type: string;
+            text: string;
+        }[];
+        isError?: undefined;
+    } | {
+        content: {
+            type: string;
+            text: string;
+        }[];
+        isError: boolean;
+    }>;
+};
+//# sourceMappingURL=get-metadata.d.ts.map

package/shared/tools/get-metadata.js

@@ -0,0 +1,63 @@
+export function getMetadata(_server, clientFactory) {
+    return {
+        name: 'get_proctor_metadata',
+        description: `Get available runtimes and exams for Proctor testing.
+
+Returns the list of available runtime environments (Docker images) and exam types
+that can be used with the run_exam tool.
+
+**Returns:**
+- runtimes: Array of runtime configurations with id, name, and Docker image
+- exams: Array of exam types with id, name, and description
+
+**Use cases:**
+- Discover available runtime environments before running an exam
+- Find the correct exam ID for a specific test type
+- Check which runtime versions are available
+- Plan which exam to run against an MCP server
+
+**Note:** Use the runtime_id and exam_id values from this response when calling run_exam.`,
+        inputSchema: {
+            type: 'object',
+            properties: {},
+            required: [],
+        },
+        handler: async () => {
+            const client = clientFactory();
+            try {
+                const response = await client.getMetadata();
+                let content = '## Available Proctor Runtimes\n\n';
+                for (const runtime of response.runtimes) {
+                    content += `- **${runtime.name}** (id: \`${runtime.id}\`)\n`;
+                    content += `  Image: \`${runtime.image}\`\n`;
+                }
+                content += '\n## Available Exams\n\n';
+                for (const exam of response.exams) {
+                    content += `- **${exam.name}** (id: \`${exam.id}\`)\n`;
+                    if (exam.description) {
+                        content += `  ${exam.description}\n`;
+                    }
+                }
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: content.trim(),
+                        },
+                    ],
+                };
+            }
+            catch (error) {
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: `Error getting Proctor metadata: ${error instanceof Error ? error.message : String(error)}`,
+                        },
+                    ],
+                    isError: true,
+                };
+            }
+        },
+    };
+}

package/shared/tools/get-prior-result.d.ts

@@ -0,0 +1,38 @@
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import type { ClientFactory } from '../server.js';
+export declare function getPriorResult(_server: Server, clientFactory: ClientFactory): {
+    name: string;
+    description: string;
+    inputSchema: {
+        type: string;
+        properties: {
+            mirror_id: {
+                type: string;
+                description: "ID of the unofficial mirror to get prior results for.";
+            };
+            exam_id: {
+                type: string;
+                description: "Exam ID to filter results by.";
+            };
+            input_json: {
+                type: string;
+                description: "Optional JSON string of the current mcp.json for matching. If provided, returns the most recent result with matching config.";
+            };
+        };
+        required: string[];
+    };
+    handler: (args: unknown) => Promise<{
+        content: {
+            type: string;
+            text: string;
+        }[];
+        isError?: undefined;
+    } | {
+        content: {
+            type: string;
+            text: string;
+        }[];
+        isError: boolean;
+    }>;
+};
+//# sourceMappingURL=get-prior-result.d.ts.map

package/shared/tools/get-prior-result.js

@@ -0,0 +1,106 @@
+import { z } from 'zod';
+// Parameter descriptions - single source of truth
+const PARAM_DESCRIPTIONS = {
+    mirror_id: 'ID of the unofficial mirror to get prior results for.',
+    exam_id: 'Exam ID to filter results by.',
+    input_json: 'Optional JSON string of the current mcp.json for matching. If provided, returns the most recent result with matching config.',
+};
+const GetPriorResultSchema = z.object({
+    mirror_id: z.number().describe(PARAM_DESCRIPTIONS.mirror_id),
+    exam_id: z.string().min(1).describe(PARAM_DESCRIPTIONS.exam_id),
+    input_json: z.string().optional().describe(PARAM_DESCRIPTIONS.input_json),
+});
+export function getPriorResult(_server, clientFactory) {
+    return {
+        name: 'get_prior_result',
+        description: `Retrieve a previous exam result for comparison.
+
+Finds the most recent prior result for the specified mirror and exam, optionally
+matching against the current input configuration.
+
+**Returns:**
+- id: Result record ID
+- datetime_performed: When the exam was run (ISO 8601)
+- results: The full exam results
+- runtime_image: Docker image used for the exam
+- match_type: "exact" if mcp.json matches exactly, "entry_key" if only entry key matches
+
+**Use cases:**
+- Compare current test results with previous runs
+- Detect regressions in MCP server functionality
+- Review historical test outcomes
+- Validate that changes haven't broken existing behavior
+
+**Note:**
+- Returns 404 if no prior result exists
+- The match_type indicates how closely the prior result matches your input_json`,
+        inputSchema: {
+            type: 'object',
+            properties: {
+                mirror_id: {
+                    type: 'number',
+                    description: PARAM_DESCRIPTIONS.mirror_id,
+                },
+                exam_id: {
+                    type: 'string',
+                    description: PARAM_DESCRIPTIONS.exam_id,
+                },
+                input_json: {
+                    type: 'string',
+                    description: PARAM_DESCRIPTIONS.input_json,
+                },
+            },
+            required: ['mirror_id', 'exam_id'],
+        },
+        handler: async (args) => {
+            const validatedArgs = GetPriorResultSchema.parse(args);
+            const client = clientFactory();
+            try {
+                const response = await client.getPriorResult({
+                    mirror_id: validatedArgs.mirror_id,
+                    exam_id: validatedArgs.exam_id,
+                    input_json: validatedArgs.input_json,
+                });
+                let content = '## Prior Result\n\n';
+                content += `**Result ID:** ${response.id}\n`;
+                content += `**Date Performed:** ${response.datetime_performed}\n`;
+                content += `**Runtime Image:** ${response.runtime_image}\n`;
+                content += `**Match Type:** ${response.match_type}\n\n`;
+                content += '### Results\n\n```json\n';
+                content += JSON.stringify(response.results, null, 2);
+                content += '\n```\n';
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: content.trim(),
+                        },
+                    ],
+                };
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                // Handle "no prior result found" as a non-error case
+                if (message.includes('No prior result found')) {
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: 'No prior result found for this mirror and exam combination.',
+                            },
+                        ],
+                    };
+                }
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: `Error getting prior result: ${message}`,
+                        },
+                    ],
+                    isError: true,
+                };
+            }
+        },
+    };
+}

package/shared/tools/run-exam.d.ts

@@ -0,0 +1,58 @@
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import type { ClientFactory } from '../server.js';
+export declare function runExam(_server: Server, clientFactory: ClientFactory): {
+    name: string;
+    description: string;
+    inputSchema: {
+        type: string;
+        properties: {
+            runtime_id: {
+                type: string;
+                description: "Runtime ID from get_proctor_metadata, or \"__custom__\" for a custom Docker image. Example: \"v0.0.37\"";
+            };
+            exam_id: {
+                type: string;
+                description: "Exam ID from get_proctor_metadata. Example: \"proctor-mcp-client-init-tools-list\" or \"proctor-mcp-client-auth-check\"";
+            };
+            mcp_config: {
+                type: string;
+                description: "JSON string of the mcp.json configuration for the MCP server. Must be a valid JSON object with server configurations.";
+            };
+            server_json: {
+                type: string;
+                description: "Optional JSON string of server.json for result enrichment. Provides additional context about the server being tested.";
+            };
+            custom_runtime_image: {
+                type: string;
+                description: "Required if runtime_id is \"__custom__\". Docker image URL in format: registry/image:tag";
+            };
+            max_retries: {
+                type: string;
+                description: "Maximum number of retry attempts (0-10). Default is 0.";
+            };
+            mcp_server_slug: {
+                type: string;
+                description: "Optional MCP server slug for auto-injection of proctor files and OAuth credentials.";
+            };
+            mcp_json_id: {
+                type: string;
+                description: "Optional McpJson ID for preloaded OAuth credentials.";
+            };
+        };
+        required: string[];
+    };
+    handler: (args: unknown) => Promise<{
+        content: {
+            type: string;
+            text: string;
+        }[];
+        isError: boolean;
+    } | {
+        content: {
+            type: string;
+            text: string;
+        }[];
+        isError?: undefined;
+    }>;
+};
+//# sourceMappingURL=run-exam.d.ts.map

package/shared/tools/run-exam.js

@@ -0,0 +1,189 @@
+import { z } from 'zod';
+// Parameter descriptions - single source of truth
+const PARAM_DESCRIPTIONS = {
+    runtime_id: 'Runtime ID from get_proctor_metadata, or "__custom__" for a custom Docker image. Example: "v0.0.37"',
+    exam_id: 'Exam ID from get_proctor_metadata. Example: "proctor-mcp-client-init-tools-list" or "proctor-mcp-client-auth-check"',
+    mcp_config: 'JSON string of the mcp.json configuration for the MCP server. Must be a valid JSON object with server configurations.',
+    server_json: 'Optional JSON string of server.json for result enrichment. Provides additional context about the server being tested.',
+    custom_runtime_image: 'Required if runtime_id is "__custom__". Docker image URL in format: registry/image:tag',
+    max_retries: 'Maximum number of retry attempts (0-10). Default is 0.',
+    mcp_server_slug: 'Optional MCP server slug for auto-injection of proctor files and OAuth credentials.',
+    mcp_json_id: 'Optional McpJson ID for preloaded OAuth credentials.',
+};
+const RunExamSchema = z.object({
+    runtime_id: z.string().min(1).describe(PARAM_DESCRIPTIONS.runtime_id),
+    exam_id: z.string().min(1).describe(PARAM_DESCRIPTIONS.exam_id),
+    mcp_config: z.string().min(1).describe(PARAM_DESCRIPTIONS.mcp_config),
+    server_json: z.string().optional().describe(PARAM_DESCRIPTIONS.server_json),
+    custom_runtime_image: z.string().optional().describe(PARAM_DESCRIPTIONS.custom_runtime_image),
+    max_retries: z.number().min(0).max(10).optional().describe(PARAM_DESCRIPTIONS.max_retries),
+    mcp_server_slug: z.string().optional().describe(PARAM_DESCRIPTIONS.mcp_server_slug),
+    mcp_json_id: z.number().optional().describe(PARAM_DESCRIPTIONS.mcp_json_id),
+});
+export function runExam(_server, clientFactory) {
+    return {
+        name: 'run_exam',
+        description: `Execute a Proctor exam against an MCP server.
+
+Runs the specified exam using the provided runtime and MCP configuration. The exam
+tests the MCP server's functionality and returns detailed results.
+
+**Returns:**
+- Streaming logs showing exam progress
+- Final result with status and detailed test outcomes
+
+**Use cases:**
+- Test an MCP server's initialization and tool listing
+- Verify authentication mechanisms work correctly
+- Run comprehensive functionality tests
+- Validate MCP protocol compliance
+- Test before publishing a new MCP server version
+
+**Note:**
+- Use get_proctor_metadata first to discover available runtimes and exams
+- The mcp_config must be a valid JSON string representing the mcp.json format
+- Custom runtime images require the "__custom__" runtime_id and custom_runtime_image parameter
+- Results can be saved using save_result for future comparison`,
+        inputSchema: {
+            type: 'object',
+            properties: {
+                runtime_id: {
+                    type: 'string',
+                    description: PARAM_DESCRIPTIONS.runtime_id,
+                },
+                exam_id: {
+                    type: 'string',
+                    description: PARAM_DESCRIPTIONS.exam_id,
+                },
+                mcp_config: {
+                    type: 'string',
+                    description: PARAM_DESCRIPTIONS.mcp_config,
+                },
+                server_json: {
+                    type: 'string',
+                    description: PARAM_DESCRIPTIONS.server_json,
+                },
+                custom_runtime_image: {
+                    type: 'string',
+                    description: PARAM_DESCRIPTIONS.custom_runtime_image,
+                },
+                max_retries: {
+                    type: 'number',
+                    description: PARAM_DESCRIPTIONS.max_retries,
+                },
+                mcp_server_slug: {
+                    type: 'string',
+                    description: PARAM_DESCRIPTIONS.mcp_server_slug,
+                },
+                mcp_json_id: {
+                    type: 'number',
+                    description: PARAM_DESCRIPTIONS.mcp_json_id,
+                },
+            },
+            required: ['runtime_id', 'exam_id', 'mcp_config'],
+        },
+        handler: async (args) => {
+            const validatedArgs = RunExamSchema.parse(args);
+            // Validate mcp_config is valid JSON
+            try {
+                JSON.parse(validatedArgs.mcp_config);
+            }
+            catch {
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: 'Error: mcp_config must be a valid JSON string',
+                        },
+                    ],
+                    isError: true,
+                };
+            }
+            // Validate custom runtime requirements
+            if (validatedArgs.runtime_id === '__custom__' && !validatedArgs.custom_runtime_image) {
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: 'Error: custom_runtime_image is required when runtime_id is "__custom__"',
+                        },
+                    ],
+                    isError: true,
+                };
+            }
+            const client = clientFactory();
+            try {
+                const logs = [];
+                let finalResult = null;
+                let errorMessage = null;
+                // Consume the streaming response
+                for await (const entry of client.runExam({
+                    runtime_id: validatedArgs.runtime_id,
+                    exam_id: validatedArgs.exam_id,
+                    mcp_config: validatedArgs.mcp_config,
+                    server_json: validatedArgs.server_json,
+                    custom_runtime_image: validatedArgs.custom_runtime_image,
+                    max_retries: validatedArgs.max_retries,
+                    mcp_server_slug: validatedArgs.mcp_server_slug,
+                    mcp_json_id: validatedArgs.mcp_json_id,
+                })) {
+                    if (entry.type === 'log') {
+                        const logData = entry.data;
+                        if (logData.message) {
+                            logs.push(`[${logData.time || 'LOG'}] ${logData.message}`);
+                        }
+                        else {
+                            logs.push(`[LOG] ${JSON.stringify(logData)}`);
+                        }
+                    }
+                    else if (entry.type === 'result') {
+                        finalResult = entry.data;
+                    }
+                    else if (entry.type === 'error') {
+                        errorMessage = entry.data.error;
+                    }
+                }
+                // Build the response
+                let content = '## Exam Execution\n\n';
+                content += `**Runtime:** ${validatedArgs.runtime_id}\n`;
+                content += `**Exam:** ${validatedArgs.exam_id}\n\n`;
+                if (logs.length > 0) {
+                    content += '### Logs\n\n```\n';
+                    content += logs.join('\n');
+                    content += '\n```\n\n';
+                }
+                if (errorMessage) {
+                    content += `### Error\n\n${errorMessage}\n`;
+                    return {
+                        content: [{ type: 'text', text: content.trim() }],
+                        isError: true,
+                    };
+                }
+                if (finalResult) {
+                    content += '### Result\n\n```json\n';
+                    content += JSON.stringify(finalResult, null, 2);
+                    content += '\n```\n';
+                }
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: content.trim(),
+                        },
+                    ],
+                };
+            }
+            catch (error) {
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: `Error running exam: ${error instanceof Error ? error.message : String(error)}`,
+                        },
+                    ],
+                    isError: true,
+                };
+            }
+        },
+    };
+}

package/shared/tools/save-result.d.ts

@@ -0,0 +1,52 @@
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import type { ClientFactory } from '../server.js';
+export declare function saveResult(_server: Server, clientFactory: ClientFactory): {
+    name: string;
+    description: string;
+    inputSchema: {
+        type: string;
+        properties: {
+            runtime_id: {
+                type: string;
+                description: "Runtime ID used for the exam, or \"__custom__\" if a custom Docker image was used.";
+            };
+            exam_id: {
+                type: string;
+                description: "Exam ID that was executed.";
+            };
+            mcp_server_slug: {
+                type: string;
+                description: "Slug of the MCP server that was tested.";
+            };
+            mirror_id: {
+                type: string;
+                description: "ID of the unofficial mirror associated with this test.";
+            };
+            results: {
+                oneOf: {
+                    type: string;
+                }[];
+                description: "Exam results as a JSON string or object. This is the full result from run_exam.";
+            };
+            custom_runtime_image: {
+                type: string;
+                description: "Required if runtime_id is \"__custom__\". The Docker image URL that was used.";
+            };
+        };
+        required: string[];
+    };
+    handler: (args: unknown) => Promise<{
+        content: {
+            type: string;
+            text: string;
+        }[];
+        isError: boolean;
+    } | {
+        content: {
+            type: string;
+            text: string;
+        }[];
+        isError?: undefined;
+    }>;
+};
+//# sourceMappingURL=save-result.d.ts.map