proctor-mcp-server 0.1.0

package/shared/tools/save-result.js

@@ -0,0 +1,122 @@
+import { z } from 'zod';
+// Parameter descriptions - single source of truth
+const PARAM_DESCRIPTIONS = {
+    runtime_id: 'Runtime ID used for the exam, or "__custom__" if a custom Docker image was used.',
+    exam_id: 'Exam ID that was executed.',
+    mcp_server_slug: 'Slug of the MCP server that was tested.',
+    mirror_id: 'ID of the unofficial mirror associated with this test.',
+    results: 'Exam results as a JSON string or object. This is the full result from run_exam.',
+    custom_runtime_image: 'Required if runtime_id is "__custom__". The Docker image URL that was used.',
+};
+const SaveResultSchema = z.object({
+    runtime_id: z.string().min(1).describe(PARAM_DESCRIPTIONS.runtime_id),
+    exam_id: z.string().min(1).describe(PARAM_DESCRIPTIONS.exam_id),
+    mcp_server_slug: z.string().min(1).describe(PARAM_DESCRIPTIONS.mcp_server_slug),
+    mirror_id: z.number().describe(PARAM_DESCRIPTIONS.mirror_id),
+    results: z.union([z.string(), z.record(z.unknown())]).describe(PARAM_DESCRIPTIONS.results),
+    custom_runtime_image: z.string().optional().describe(PARAM_DESCRIPTIONS.custom_runtime_image),
+});
+export function saveResult(_server, clientFactory) {
+    return {
+        name: 'save_result',
+        description: `Save exam results to the database for future comparison.
+
+Stores the results of a Proctor exam run so they can be retrieved later for
+comparison with new test runs.
+
+**Returns:**
+- success: boolean indicating if the save was successful
+- id: ID of the saved result record
+
+**Use cases:**
+- Persist exam results after running tests
+- Create a baseline for future comparisons
+- Track test history for an MCP server
+- Enable regression testing by comparing against prior results
+
+**Note:**
+- The mirror_id must be a valid unofficial mirror ID
+- Results should be the full output from run_exam
+- Custom runtime images require the custom_runtime_image parameter`,
+        inputSchema: {
+            type: 'object',
+            properties: {
+                runtime_id: {
+                    type: 'string',
+                    description: PARAM_DESCRIPTIONS.runtime_id,
+                },
+                exam_id: {
+                    type: 'string',
+                    description: PARAM_DESCRIPTIONS.exam_id,
+                },
+                mcp_server_slug: {
+                    type: 'string',
+                    description: PARAM_DESCRIPTIONS.mcp_server_slug,
+                },
+                mirror_id: {
+                    type: 'number',
+                    description: PARAM_DESCRIPTIONS.mirror_id,
+                },
+                results: {
+                    oneOf: [{ type: 'string' }, { type: 'object' }],
+                    description: PARAM_DESCRIPTIONS.results,
+                },
+                custom_runtime_image: {
+                    type: 'string',
+                    description: PARAM_DESCRIPTIONS.custom_runtime_image,
+                },
+            },
+            required: ['runtime_id', 'exam_id', 'mcp_server_slug', 'mirror_id', 'results'],
+        },
+        handler: async (args) => {
+            const validatedArgs = SaveResultSchema.parse(args);
+            // Validate custom runtime requirements
+            if (validatedArgs.runtime_id === '__custom__' && !validatedArgs.custom_runtime_image) {
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: 'Error: custom_runtime_image is required when runtime_id is "__custom__"',
+                        },
+                    ],
+                    isError: true,
+                };
+            }
+            const client = clientFactory();
+            try {
+                const response = await client.saveResult({
+                    runtime_id: validatedArgs.runtime_id,
+                    exam_id: validatedArgs.exam_id,
+                    mcp_server_slug: validatedArgs.mcp_server_slug,
+                    mirror_id: validatedArgs.mirror_id,
+                    results: validatedArgs.results,
+                    custom_runtime_image: validatedArgs.custom_runtime_image,
+                });
+                let content = '## Result Saved\n\n';
+                content += `**Success:** ${response.success}\n`;
+                content += `**Result ID:** ${response.id}\n\n`;
+                content +=
+                    'The exam result has been saved and can be retrieved for comparison using get_prior_result.';
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: content.trim(),
+                        },
+                    ],
+                };
+            }
+            catch (error) {
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: `Error saving result: ${error instanceof Error ? error.message : String(error)}`,
+                        },
+                    ],
+                    isError: true,
+                };
+            }
+        },
+    };
+}

package/shared/tools.d.ts

@@ -0,0 +1,44 @@
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { ClientFactory } from './server.js';
+/**
+ * Tool group definitions - groups of related tools that can be enabled/disabled together
+ *
+ * Each group has two variants:
+ * - Base group (e.g., 'exams'): Includes all tools (read + write operations)
+ * - Readonly group (e.g., 'exams_readonly'): Includes only read operations
+ *
+ * Groups:
+ * - exams / exams_readonly: Exam execution and result management tools
+ * - machines / machines_readonly: Fly.io machine management tools
+ */
+export type ToolGroup = 'exams' | 'exams_readonly' | 'machines' | 'machines_readonly';
+/**
+ * Parse enabled tool groups from environment variable or parameter
+ * @param enabledGroupsParam - Comma-separated list of tool groups (e.g., "exams,machines_readonly")
+ * @returns Array of enabled tool groups
+ */
+export declare function parseEnabledToolGroups(enabledGroupsParam?: string): ToolGroup[];
+/**
+ * Creates a function to register all tools with the server.
+ * This pattern uses individual tool files for better modularity and testability.
+ *
+ * Each tool is defined in its own file under the `tools/` directory and follows
+ * a factory pattern that accepts the server and clientFactory as parameters.
+ *
+ * Tool groups can be enabled/disabled via the TOOL_GROUPS environment variable
+ * (comma-separated list, e.g., "exams,machines_readonly"). If not set, all
+ * base tool groups are enabled by default (full read+write access).
+ *
+ * Available tool groups:
+ * - exams: All exam-related tools (read + write)
+ * - exams_readonly: Exam tools (read only - get_proctor_metadata, get_prior_result)
+ * - machines: All machine management tools (read + write)
+ * - machines_readonly: Machine tools (read only - get_machines)
+ *
+ * @param clientFactory - Factory function that creates client instances
+ * @param enabledGroups - Optional comma-separated list of enabled tool groups (overrides env var)
+ * @returns Function that registers all tools with a server
+ */
+export declare function createRegisterTools(clientFactory: ClientFactory, enabledGroups?: string): (server: Server) => void;
+export declare function registerTools(server: Server): void;
+//# sourceMappingURL=tools.d.ts.map

package/shared/tools.js

@@ -0,0 +1,128 @@
+import { ListToolsRequestSchema, CallToolRequestSchema } from '@modelcontextprotocol/sdk/types.js';
+import { getMetadata } from './tools/get-metadata.js';
+import { runExam } from './tools/run-exam.js';
+import { saveResult } from './tools/save-result.js';
+import { getPriorResult } from './tools/get-prior-result.js';
+import { getMachines } from './tools/get-machines.js';
+import { destroyMachine } from './tools/destroy-machine.js';
+import { cancelExam } from './tools/cancel-exam.js';
+const ALL_TOOLS = [
+    // Exam tools
+    { factory: getMetadata, group: 'exams', isWriteOperation: false },
+    { factory: runExam, group: 'exams', isWriteOperation: true },
+    { factory: saveResult, group: 'exams', isWriteOperation: true },
+    { factory: getPriorResult, group: 'exams', isWriteOperation: false },
+    // Machine management tools
+    { factory: getMachines, group: 'machines', isWriteOperation: false },
+    { factory: destroyMachine, group: 'machines', isWriteOperation: true },
+    { factory: cancelExam, group: 'machines', isWriteOperation: true },
+];
+/**
+ * All valid tool groups (base groups and their _readonly variants)
+ */
+const VALID_TOOL_GROUPS = ['exams', 'exams_readonly', 'machines', 'machines_readonly'];
+/**
+ * Base groups (without _readonly suffix) - used for default "all groups" behavior
+ */
+const BASE_TOOL_GROUPS = ['exams', 'machines'];
+/**
+ * Parse enabled tool groups from environment variable or parameter
+ * @param enabledGroupsParam - Comma-separated list of tool groups (e.g., "exams,machines_readonly")
+ * @returns Array of enabled tool groups
+ */
+export function parseEnabledToolGroups(enabledGroupsParam) {
+    const groupsStr = enabledGroupsParam || process.env.TOOL_GROUPS || '';
+    if (!groupsStr) {
+        // Default: all base groups enabled (full read+write access)
+        return [...BASE_TOOL_GROUPS];
+    }
+    const groups = groupsStr.split(',').map((g) => g.trim());
+    const validGroups = [];
+    for (const group of groups) {
+        if (VALID_TOOL_GROUPS.includes(group) &&
+            !validGroups.includes(group)) {
+            validGroups.push(group);
+        }
+        else if (!VALID_TOOL_GROUPS.includes(group)) {
+            console.warn(`Unknown tool group: ${group}`);
+        }
+    }
+    return validGroups;
+}
+/**
+ * Check if a tool should be included based on enabled groups
+ * @param toolDef - The tool definition to check
+ * @param enabledGroups - Array of enabled tool groups
+ * @returns true if the tool should be included
+ */
+function shouldIncludeTool(toolDef, enabledGroups) {
+    const baseGroup = toolDef.group;
+    const readonlyGroup = `${baseGroup}_readonly`;
+    // Check if the base group (full access) is enabled
+    if (enabledGroups.includes(baseGroup)) {
+        return true;
+    }
+    // Check if the readonly group is enabled (only include read operations)
+    if (enabledGroups.includes(readonlyGroup) && !toolDef.isWriteOperation) {
+        return true;
+    }
+    return false;
+}
+/**
+ * Creates a function to register all tools with the server.
+ * This pattern uses individual tool files for better modularity and testability.
+ *
+ * Each tool is defined in its own file under the `tools/` directory and follows
+ * a factory pattern that accepts the server and clientFactory as parameters.
+ *
+ * Tool groups can be enabled/disabled via the TOOL_GROUPS environment variable
+ * (comma-separated list, e.g., "exams,machines_readonly"). If not set, all
+ * base tool groups are enabled by default (full read+write access).
+ *
+ * Available tool groups:
+ * - exams: All exam-related tools (read + write)
+ * - exams_readonly: Exam tools (read only - get_proctor_metadata, get_prior_result)
+ * - machines: All machine management tools (read + write)
+ * - machines_readonly: Machine tools (read only - get_machines)
+ *
+ * @param clientFactory - Factory function that creates client instances
+ * @param enabledGroups - Optional comma-separated list of enabled tool groups (overrides env var)
+ * @returns Function that registers all tools with a server
+ */
+export function createRegisterTools(clientFactory, enabledGroups) {
+    return (server) => {
+        const enabledToolGroups = parseEnabledToolGroups(enabledGroups);
+        // Filter tools based on enabled groups
+        const enabledTools = ALL_TOOLS.filter((toolDef) => shouldIncludeTool(toolDef, enabledToolGroups));
+        // Create tool instances
+        const tools = enabledTools.map((toolDef) => toolDef.factory(server, clientFactory));
+        // List available tools
+        server.setRequestHandler(ListToolsRequestSchema, async () => {
+            return {
+                tools: tools.map((tool) => ({
+                    name: tool.name,
+                    description: tool.description,
+                    inputSchema: tool.inputSchema,
+                })),
+            };
+        });
+        // Handle tool calls
+        server.setRequestHandler(CallToolRequestSchema, async (request) => {
+            const { name, arguments: args } = request.params;
+            const tool = tools.find((t) => t.name === name);
+            if (!tool) {
+                throw new Error(`Unknown tool: ${name}`);
+            }
+            return await tool.handler(args);
+        });
+    };
+}
+// Keep the original registerTools for backward compatibility
+export function registerTools(server) {
+    // This maintains compatibility but doesn't use dependency injection
+    const factory = () => {
+        throw new Error('No client factory provided - use createRegisterTools for dependency injection');
+    };
+    const register = createRegisterTools(factory);
+    register(server);
+}

package/shared/types.d.ts

@@ -0,0 +1,151 @@
+/**
+ * Type definitions for Proctor MCP Server
+ */
+/**
+ * Runtime configuration for running Proctor exams
+ */
+export interface ProctorRuntime {
+    id: string;
+    name: string;
+    image: string;
+}
+/**
+ * Exam configuration available in Proctor
+ */
+export interface ProctorExam {
+    id: string;
+    name: string;
+    description: string;
+}
+/**
+ * Metadata response from the Proctor API
+ */
+export interface ProctorMetadataResponse {
+    runtimes: ProctorRuntime[];
+    exams: ProctorExam[];
+}
+/**
+ * Log entry from a running exam
+ */
+export interface ExamLogEntry {
+    time?: string;
+    message?: string;
+    [key: string]: unknown;
+}
+/**
+ * Streaming response types from run_exam endpoint
+ */
+export interface ExamStreamLog {
+    type: 'log';
+    data: ExamLogEntry;
+}
+export interface ExamStreamResult {
+    type: 'result';
+    data: ExamResult;
+}
+export interface ExamStreamError {
+    type: 'error';
+    data: {
+        error: string;
+    };
+}
+export type ExamStreamEntry = ExamStreamLog | ExamStreamResult | ExamStreamError;
+/**
+ * Final exam result
+ */
+export interface ExamResult {
+    status?: string;
+    input?: {
+        'mcp.json'?: Record<string, unknown>;
+        'server.json'?: Record<string, unknown>;
+    };
+    [key: string]: unknown;
+}
+/**
+ * Parameters for running an exam
+ */
+export interface RunExamParams {
+    runtime_id: string;
+    exam_id: string;
+    mcp_config: string;
+    server_json?: string;
+    custom_runtime_image?: string;
+    max_retries?: number;
+    mcp_server_slug?: string;
+    mcp_json_id?: number;
+}
+/**
+ * Parameters for saving exam results
+ */
+export interface SaveResultParams {
+    runtime_id: string;
+    exam_id: string;
+    mcp_server_slug: string;
+    mirror_id: number;
+    results: string | Record<string, unknown>;
+    custom_runtime_image?: string;
+}
+/**
+ * Response from save_result endpoint
+ */
+export interface SaveResultResponse {
+    success: boolean;
+    id: number;
+}
+/**
+ * Parameters for getting prior results
+ */
+export interface PriorResultParams {
+    mirror_id: number;
+    exam_id: string;
+    input_json?: string;
+}
+/**
+ * Response from prior_result endpoint
+ */
+export interface PriorResultResponse {
+    id: number;
+    datetime_performed: string;
+    results: ExamResult;
+    runtime_image: string;
+    match_type: 'exact' | 'entry_key';
+}
+/**
+ * Fly.io machine information
+ */
+export interface FlyMachine {
+    id: string;
+    name?: string;
+    state?: string;
+    region?: string;
+    created_at?: string;
+    [key: string]: unknown;
+}
+/**
+ * Response from machines endpoint
+ */
+export interface MachinesResponse {
+    machines: FlyMachine[];
+}
+/**
+ * Parameters for canceling an exam
+ */
+export interface CancelExamParams {
+    machine_id: string;
+    exam_id: string;
+}
+/**
+ * Response from cancel_exam endpoint
+ */
+export interface CancelExamResponse {
+    success?: boolean;
+    message?: string;
+    [key: string]: unknown;
+}
+/**
+ * Error response from API
+ */
+export interface ApiError {
+    error: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/shared/types.js

@@ -0,0 +1,4 @@
+/**
+ * Type definitions for Proctor MCP Server
+ */
+export {};