webmcp-cli 1.0.0 → 1.1.0

package/dist/agent/features/agent-simulator.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Agent Simulator
+ *
+ * Simulates AI agent interactions with WebMCP tools using function calling.
+ * Tests whether an LLM can correctly route to tools and extract parameters.
+ */
+import { LLMClient, type LLMToolCall } from '../llm-client.js';
+import type { DetectedTool } from '../../core/types/tool.js';
+import type { TestCase } from './test-case-generator.js';
+export interface SimulationConfig {
+    maxRounds: number;
+    mockToolResponses: boolean;
+    model: string;
+    siteContext: string;
+    maxCost?: number;
+}
+export interface SimulationRound {
+    roundNumber: number;
+    userMessage: string;
+    agentResponse: {
+        text: string | null;
+        toolCalls: LLMToolCall[];
+    };
+    toolResults: ToolResult[];
+    evaluation: RoundEvaluation;
+}
+export interface ToolResult {
+    toolName: string;
+    input: Record<string, unknown>;
+    output: string;
+    success: boolean;
+}
+export interface RoundEvaluation {
+    correctToolSelected: boolean | null;
+    parameterAccuracy: number;
+    hallucinatedParams: string[];
+    missingRequiredParams: string[];
+    unnecessaryRound: boolean;
+}
+export interface SimulationResult {
+    testCase: TestCase;
+    model: string;
+    rounds: SimulationRound[];
+    totalRounds: number;
+    toolInvocations: number;
+    correctToolOnFirstTry: boolean;
+    finalSuccess: boolean;
+    totalLatencyMs: number;
+    totalCost: number;
+    evaluation: {
+        routingAccuracy: number;
+        parameterAccuracy: number;
+        hallucinationRate: number;
+        roundEfficiency: number;
+        overallScore: number;
+        failureReason: string | null;
+    };
+}
+/**
+ * Run a single simulation for a test case
+ */
+export declare function runSimulation(testCase: TestCase, tools: DetectedTool[], config: SimulationConfig, llm?: LLMClient): Promise<SimulationResult>;
+/**
+ * Run simulations for multiple test cases
+ */
+export declare function runSimulations(testCases: TestCase[], tools: DetectedTool[], config: SimulationConfig, onProgress?: (completed: number, total: number) => void): Promise<SimulationResult[]>;
+export default runSimulation;

package/dist/agent/features/agent-simulator.js

@@ -0,0 +1,368 @@
+/**
+ * Agent Simulator
+ *
+ * Simulates AI agent interactions with WebMCP tools using function calling.
+ * Tests whether an LLM can correctly route to tools and extract parameters.
+ */
+import { LLMClient, createLLMClient } from '../llm-client.js';
+// ═══════════════════════════════════════════════════════
+// AGENT SYSTEM PROMPT
+// ═══════════════════════════════════════════════════════
+function buildAgentSystemPrompt(siteContext) {
+    return `You are an AI assistant integrated into a web browser. The user is currently viewing a ${siteContext} website. You have access to the website's functionality through the tools provided.
+
+BEHAVIOR RULES:
+1. When the user asks you to do something, check if any of the available tools can help accomplish their goal.
+2. If a tool can help, invoke it with the correct parameters extracted from the user's request.
+3. If required parameters are missing from the user's request, ask the user for them — do NOT guess or hallucinate values.
+4. If optional parameters are missing, you may omit them or use reasonable defaults.
+5. If no tool matches the user's request, tell them honestly that you can't help with that specific task on this website.
+6. If the user's request is ambiguous and could match multiple tools, ask for clarification.
+7. After invoking a tool, summarize the result for the user in natural language.
+
+IMPORTANT:
+- Only use the tools you've been given. Do NOT make up tools that don't exist.
+- Use the exact parameter names and types defined in each tool's schema.
+- Do NOT provide parameters that the user didn't mention or that you can't reasonably infer.
+- If a parameter expects specific formats (like airport codes, dates), try to convert the user's input to that format.`;
+}
+// ═══════════════════════════════════════════════════════
+// MOCK RESPONSE GENERATOR
+// ═══════════════════════════════════════════════════════
+function generateMockResponse(toolCall, tools) {
+    const tool = tools.find(t => t.name === toolCall.toolName);
+    if (!tool) {
+        return JSON.stringify({
+            error: 'Tool not found',
+            message: `Unknown tool: ${toolCall.toolName}`,
+        });
+    }
+    // Generate a plausible mock response based on tool name
+    const name = toolCall.toolName.toLowerCase();
+    if (name.includes('search') || name.includes('find') || name.includes('list')) {
+        return JSON.stringify({
+            results: [
+                { id: 'result-1', name: 'Example Result 1', description: 'First result' },
+                { id: 'result-2', name: 'Example Result 2', description: 'Second result' },
+                { id: 'result-3', name: 'Example Result 3', description: 'Third result' },
+            ],
+            total: 3,
+            message: `Found 3 results for your query`,
+        });
+    }
+    if (name.includes('create') || name.includes('add') || name.includes('book')) {
+        return JSON.stringify({
+            success: true,
+            id: 'new-item-123',
+            message: 'Successfully created new item',
+            details: toolCall.toolInput,
+        });
+    }
+    if (name.includes('get') || name.includes('fetch') || name.includes('retrieve')) {
+        return JSON.stringify({
+            id: toolCall.toolInput['id'] || 'item-1',
+            name: 'Retrieved Item',
+            status: 'active',
+            createdAt: '2026-01-15T10:00:00Z',
+        });
+    }
+    if (name.includes('update') || name.includes('edit') || name.includes('modify')) {
+        return JSON.stringify({
+            success: true,
+            message: 'Successfully updated item',
+            changes: toolCall.toolInput,
+        });
+    }
+    if (name.includes('delete') || name.includes('remove') || name.includes('cancel')) {
+        return JSON.stringify({
+            success: true,
+            message: 'Successfully deleted item',
+            id: toolCall.toolInput['id'] || 'deleted-item',
+        });
+    }
+    // Default response
+    return JSON.stringify({
+        success: true,
+        message: `Tool ${toolCall.toolName} executed successfully`,
+        input: toolCall.toolInput,
+    });
+}
+// ═══════════════════════════════════════════════════════
+// EVALUATION FUNCTIONS
+// ═══════════════════════════════════════════════════════
+function evaluateRound(roundNumber, toolCalls, testCase, _tools) {
+    const expected = testCase.expectedBehavior;
+    // No tool calls
+    if (toolCalls.length === 0) {
+        return {
+            correctToolSelected: expected.shouldInvokeTool ? false : null,
+            parameterAccuracy: 0,
+            hallucinatedParams: [],
+            missingRequiredParams: Object.keys(expected.expectedParams),
+            unnecessaryRound: roundNumber > 1,
+        };
+    }
+    // Check first tool call (primary action)
+    const firstCall = toolCalls[0];
+    if (!firstCall) {
+        return {
+            correctToolSelected: expected.shouldInvokeTool ? false : null,
+            parameterAccuracy: 0,
+            hallucinatedParams: [],
+            missingRequiredParams: Object.keys(expected.expectedParams),
+            unnecessaryRound: roundNumber > 1,
+        };
+    }
+    const correctTool = firstCall.toolName === expected.toolName;
+    // Evaluate parameter accuracy
+    const expectedParams = expected.expectedParams;
+    const actualParams = firstCall.toolInput;
+    let correctParams = 0;
+    let totalExpected = Object.keys(expectedParams).length;
+    const hallucinatedParams = [];
+    const missingParams = [];
+    // Check expected params
+    for (const [key, value] of Object.entries(expectedParams)) {
+        if (key in actualParams) {
+            // Simple equality check (could be more sophisticated)
+            if (actualParams[key] === value ||
+                String(actualParams[key]).toLowerCase() === String(value).toLowerCase()) {
+                correctParams++;
+            }
+        }
+        else {
+            missingParams.push(key);
+        }
+    }
+    // Check for hallucinated params (params not in expected)
+    for (const key of Object.keys(actualParams)) {
+        if (!(key in expectedParams)) {
+            // Only flag if this param wasn't provided by user
+            hallucinatedParams.push(key);
+        }
+    }
+    const paramAccuracy = totalExpected > 0 ? correctParams / totalExpected : 1;
+    return {
+        correctToolSelected: correctTool,
+        parameterAccuracy: paramAccuracy,
+        hallucinatedParams,
+        missingRequiredParams: missingParams,
+        unnecessaryRound: roundNumber > 1 && correctTool && paramAccuracy > 0.8,
+    };
+}
+function buildSimulationResult(testCase, model, rounds, totalLatency, totalCost) {
+    // Aggregate metrics
+    const toolInvocations = rounds.reduce((sum, r) => sum + r.agentResponse.toolCalls.length, 0);
+    const correctToolOnFirstTry = rounds[0]?.evaluation.correctToolSelected === true;
+    // Check if final result was successful
+    const lastRound = rounds[rounds.length - 1];
+    const expected = testCase.expectedBehavior;
+    let finalSuccess = false;
+    let failureReason = null;
+    if (!expected.shouldInvokeTool) {
+        // Out of scope - success if no tool was called
+        finalSuccess = toolInvocations === 0;
+        if (!finalSuccess) {
+            failureReason = 'Called a tool when none should have been used';
+        }
+    }
+    else {
+        // Should invoke tool - check if correct tool was ever called
+        const correctToolCalled = rounds.some(r => r.agentResponse.toolCalls.some(tc => tc.toolName === expected.toolName));
+        if (!correctToolCalled) {
+            failureReason = `Never called expected tool: ${expected.toolName}`;
+        }
+        else {
+            // Check parameter accuracy of the correct call
+            const correctCallRound = rounds.find(r => r.agentResponse.toolCalls.some(tc => tc.toolName === expected.toolName));
+            if (correctCallRound && correctCallRound.evaluation.parameterAccuracy >= 0.5) {
+                finalSuccess = true;
+            }
+            else {
+                failureReason = 'Called correct tool but with wrong parameters';
+            }
+        }
+    }
+    // Calculate aggregate scores
+    const routingAccuracy = correctToolOnFirstTry ? 1 :
+        rounds.some(r => r.evaluation.correctToolSelected) ? 0.5 : 0;
+    const avgParamAccuracy = rounds.length > 0
+        ? rounds.reduce((sum, r) => sum + r.evaluation.parameterAccuracy, 0) / rounds.length
+        : 0;
+    const totalHallucinated = rounds.reduce((sum, r) => sum + r.evaluation.hallucinatedParams.length, 0);
+    const totalParams = rounds.reduce((sum, r) => sum + Object.keys(r.agentResponse.toolCalls[0]?.toolInput || {}).length, 0);
+    const hallucinationRate = totalParams > 0 ? totalHallucinated / totalParams : 0;
+    // Round efficiency: 1.0 = solved in 1 round, less for more rounds
+    const optimalRounds = 1;
+    const roundEfficiency = optimalRounds / Math.max(rounds.length, 1);
+    // Overall score (weighted)
+    const overallScore = Math.round((routingAccuracy * 40 +
+        avgParamAccuracy * 30 +
+        (1 - hallucinationRate) * 15 +
+        roundEfficiency * 15) * 100 / 100);
+    return {
+        testCase,
+        model,
+        rounds,
+        totalRounds: rounds.length,
+        toolInvocations,
+        correctToolOnFirstTry,
+        finalSuccess,
+        totalLatencyMs: totalLatency,
+        totalCost,
+        evaluation: {
+            routingAccuracy,
+            parameterAccuracy: avgParamAccuracy,
+            hallucinationRate,
+            roundEfficiency,
+            overallScore,
+            failureReason,
+        },
+    };
+}
+// ═══════════════════════════════════════════════════════
+// SIMULATION RUNNER
+// ═══════════════════════════════════════════════════════
+/**
+ * Run a single simulation for a test case
+ */
+export async function runSimulation(testCase, tools, config, llm) {
+    const client = llm || createLLMClient({ model: config.model });
+    // Convert tools to LLM format
+    const llmTools = tools.map(t => ({
+        name: t.name,
+        description: t.description,
+        input_schema: t.inputSchema || { type: 'object', properties: {} },
+    }));
+    const messages = [
+        { role: 'system', content: buildAgentSystemPrompt(config.siteContext) },
+    ];
+    const rounds = [];
+    let currentMessage = testCase.prompt;
+    let totalLatency = 0;
+    let totalCost = 0;
+    let isComplete = false;
+    for (let round = 1; round <= config.maxRounds && !isComplete; round++) {
+        // Add user message
+        messages.push({ role: 'user', content: currentMessage });
+        try {
+            // Get agent response with tools
+            const response = await client.chatWithTools(messages, llmTools);
+            totalLatency += response.latencyMs;
+            totalCost += response.cost;
+            // Check cost limit
+            if (config.maxCost && totalCost >= config.maxCost) {
+                break;
+            }
+            // Process tool calls
+            const toolResults = [];
+            if (response.toolCalls.length > 0) {
+                for (const toolCall of response.toolCalls) {
+                    // Add assistant message with tool call
+                    messages.push({
+                        role: 'assistant',
+                        content: null,
+                        tool_calls: [{
+                                id: `call_${round}_${toolCall.toolName}`,
+                                type: 'function',
+                                function: {
+                                    name: toolCall.toolName,
+                                    arguments: JSON.stringify(toolCall.toolInput),
+                                },
+                            }],
+                    });
+                    // Generate mock tool response
+                    const toolOutput = config.mockToolResponses
+                        ? generateMockResponse(toolCall, tools)
+                        : JSON.stringify({ error: 'Real tool execution not implemented' });
+                    toolResults.push({
+                        toolName: toolCall.toolName,
+                        input: toolCall.toolInput,
+                        output: toolOutput,
+                        success: true,
+                    });
+                    // Add tool result to conversation
+                    messages.push({
+                        role: 'tool',
+                        content: toolOutput,
+                        tool_call_id: `call_${round}_${toolCall.toolName}`,
+                    });
+                }
+                // Get agent's follow-up response
+                const followUp = await client.chatWithTools(messages, llmTools);
+                totalLatency += followUp.latencyMs;
+                totalCost += followUp.cost;
+                messages.push({ role: 'assistant', content: followUp.text });
+                // Complete if no more tool calls
+                if (followUp.toolCalls.length === 0) {
+                    isComplete = true;
+                }
+                rounds.push({
+                    roundNumber: round,
+                    userMessage: currentMessage,
+                    agentResponse: { text: followUp.text, toolCalls: response.toolCalls },
+                    toolResults,
+                    evaluation: evaluateRound(round, response.toolCalls, testCase, tools),
+                });
+            }
+            else {
+                // Agent responded with text only
+                messages.push({ role: 'assistant', content: response.text });
+                rounds.push({
+                    roundNumber: round,
+                    userMessage: currentMessage,
+                    agentResponse: { text: response.text, toolCalls: [] },
+                    toolResults: [],
+                    evaluation: evaluateRound(round, [], testCase, tools),
+                });
+                // If agent asked a clarifying question, we'd need to respond
+                // For now, we complete after text-only response
+                isComplete = true;
+            }
+        }
+        catch (error) {
+            // Handle API errors gracefully
+            rounds.push({
+                roundNumber: round,
+                userMessage: currentMessage,
+                agentResponse: { text: `Error: ${error instanceof Error ? error.message : 'Unknown error'}`, toolCalls: [] },
+                toolResults: [],
+                evaluation: {
+                    correctToolSelected: null,
+                    parameterAccuracy: 0,
+                    hallucinatedParams: [],
+                    missingRequiredParams: [],
+                    unnecessaryRound: false,
+                },
+            });
+            isComplete = true;
+        }
+    }
+    return buildSimulationResult(testCase, config.model, rounds, totalLatency, totalCost);
+}
+/**
+ * Run simulations for multiple test cases
+ */
+export async function runSimulations(testCases, tools, config, onProgress) {
+    const results = [];
+    const llm = createLLMClient({ model: config.model });
+    let totalCost = 0;
+    for (let i = 0; i < testCases.length; i++) {
+        // Check cost limit
+        if (config.maxCost && totalCost >= config.maxCost) {
+            break;
+        }
+        const testCase = testCases[i];
+        if (!testCase)
+            continue;
+        const result = await runSimulation(testCase, tools, config, llm);
+        results.push(result);
+        totalCost += result.totalCost;
+        if (onProgress) {
+            onProgress(i + 1, testCases.length);
+        }
+    }
+    return results;
+}
+export default runSimulation;
+//# sourceMappingURL=agent-simulator.js.map

package/dist/agent/features/agent-simulator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-simulator.js","sourceRoot":"","sources":["../../../src/agent/features/agent-simulator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,SAAS,EAAE,eAAe,EAAmD,MAAM,kBAAkB,CAAC;AA8D/G,0DAA0D;AAC1D,sBAAsB;AACtB,0DAA0D;AAE1D,SAAS,sBAAsB,CAAC,WAAmB;IACjD,OAAO,0FAA0F,WAAW;;;;;;;;;;;;;;;uHAeS,CAAC;AACxH,CAAC;AAED,0DAA0D;AAC1D,0BAA0B;AAC1B,0DAA0D;AAE1D,SAAS,oBAAoB,CAC3B,QAAqB,EACrB,KAAqB;IAErB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC3D,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;YACpB,KAAK,EAAE,gBAAgB;YACvB,OAAO,EAAE,iBAAiB,QAAQ,CAAC,QAAQ,EAAE;SAC9C,CAAC,CAAC;IACL,CAAC;IAED,wDAAwD;IACxD,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAE7C,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9E,OAAO,IAAI,CAAC,SAAS,CAAC;YACpB,OAAO,EAAE;gBACP,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,kBAAkB,EAAE,WAAW,EAAE,cAAc,EAAE;gBACzE,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,kBAAkB,EAAE,WAAW,EAAE,eAAe,EAAE;gBAC1E,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,kBAAkB,EAAE,WAAW,EAAE,cAAc,EAAE;aAC1E;YACD,KAAK,EAAE,CAAC;YACR,OAAO,EAAE,gCAAgC;SAC1C,CAAC,CAAC;IACL,CAAC;IAED,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7E,OAAO,IAAI,CAAC,SAAS,CAAC;YACpB,OAAO,EAAE,IAAI;YACb,EAAE,EAAE,cAAc;YAClB,OAAO,EAAE,+BAA+B;YACxC,OAAO,EAAE,QAAQ,CAAC,SAAS;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAChF,OAAO,IAAI,CAAC,SAAS,CAAC;YACpB,EAAE,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,QAAQ;YACxC,IAAI,EAAE,gBAAgB;YACtB,MAAM,EAAE,QAAQ;YAChB,SAAS,EAAE,sBAAsB;SAClC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChF,OAAO,IAAI,CAAC,SAAS,CAAC;YACpB,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,2BAA2B;YACpC,OAAO,EAAE,QAAQ,CAAC,SAAS;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAClF,OAAO,IAAI,CAAC,SAAS,CAAC;YACpB,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,2BAA2B;YACpC,EAAE,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,cAAc;SAC/C,CAAC,CAAC;IACL,CAAC;IAED,mBAAmB;IACnB,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,QAAQ,QAAQ,CAAC,QAAQ,wBAAwB;QAC1D,KAAK,EAAE,QAAQ,CAAC,SAAS;KAC1B,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,uBAAuB;AACvB,0DAA0D;AAE1D,SAAS,aAAa,CACpB,WAAmB,EACnB,SAAwB,EACxB,QAAkB,EAClB,MAAsB;IAEtB,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC;IAE3C,gBAAgB;IAChB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO;YACL,mBAAmB,EAAE,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;YAC7D,iBAAiB,EAAE,CAAC;YACpB,kBAAkB,EAAE,EAAE;YACtB,qBAAqB,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC;YAC3D,gBAAgB,EAAE,WAAW,GAAG,CAAC;SAClC,CAAC;IACJ,CAAC;IAED,yCAAyC;IACzC,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAC/B,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO;YACL,mBAAmB,EAAE,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;YAC7D,iBAAiB,EAAE,CAAC;YACpB,kBAAkB,EAAE,EAAE;YACtB,qBAAqB,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC;YAC3D,gBAAgB,EAAE,WAAW,GAAG,CAAC;SAClC,CAAC;IACJ,CAAC;IACD,MAAM,WAAW,GAAG,SAAS,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ,CAAC;IAE7D,8BAA8B;IAC9B,MAAM,cAAc,GAAG,QAAQ,CAAC,cAAc,CAAC;IAC/C,MAAM,YAAY,GAAG,SAAS,CAAC,SAAS,CAAC;IAEzC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC;IACvD,MAAM,kBAAkB,GAAa,EAAE,CAAC;IACxC,MAAM,aAAa,GAAa,EAAE,CAAC;IAEnC,wBAAwB;IACxB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;QAC1D,IAAI,GAAG,IAAI,YAAY,EAAE,CAAC;YACxB,sDAAsD;YACtD,IAAI,YAAY,CAAC,GAAG,CAAC,KAAK,KAAK;gBAC3B,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;gBAC5E,aAAa,EAAE,CAAC;YAClB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,yDAAyD;IACzD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;QAC5C,IAAI,CAAC,CAAC,GAAG,IAAI,cAAc,CAAC,EAAE,CAAC;YAC7B,kDAAkD;YAClD,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5E,OAAO;QACL,mBAAmB,EAAE,WAAW;QAChC,iBAAiB,EAAE,aAAa;QAChC,kBAAkB;QAClB,qBAAqB,EAAE,aAAa;QACpC,gBAAgB,EAAE,WAAW,GAAG,CAAC,IAAI,WAAW,IAAI,aAAa,GAAG,GAAG;KACxE,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAC5B,QAAkB,EAClB,KAAa,EACb,MAAyB,EACzB,YAAoB,EACpB,SAAiB;IAEjB,oBAAoB;IACpB,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CACnC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CACtD,CAAC;IAEF,MAAM,qBAAqB,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,mBAAmB,KAAK,IAAI,CAAC;IAEjF,uCAAuC;IACvC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC;IAE3C,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,aAAa,GAAkB,IAAI,CAAC;IAExC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;QAC/B,+CAA+C;QAC/C,YAAY,GAAG,eAAe,KAAK,CAAC,CAAC;QACrC,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,aAAa,GAAG,+CAA+C,CAAC;QAClE,CAAC;IACH,CAAC;SAAM,CAAC;QACN,6DAA6D;QAC7D,MAAM,iBAAiB,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CACxC,CAAC,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ,CAAC,CACxE,CAAC;QAEF,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACvB,aAAa,GAAG,+BAA+B,QAAQ,CAAC,QAAQ,EAAE,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,+CAA+C;YAC/C,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CACvC,CAAC,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ,CAAC,CACxE,CAAC;YAEF,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,UAAU,CAAC,iBAAiB,IAAI,GAAG,EAAE,CAAC;gBAC7E,YAAY,GAAG,IAAI,CAAC;YACtB,CAAC;iBAAM,CAAC;gBACN,aAAa,GAAG,+CAA+C,CAAC;YAClE,CAAC;QACH,CAAC;IACH,CAAC;IAED,6BAA6B;IAC7B,MAAM,eAAe,GAAG,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/D,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC;QACxC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,iBAAiB,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM;QACpF,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,CACrC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,CAC5D,CAAC;IACF,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAC/B,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CACvF,CAAC;IACF,MAAM,iBAAiB,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAEhF,kEAAkE;IAClE,MAAM,aAAa,GAAG,CAAC,CAAC;IACxB,MAAM,eAAe,GAAG,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAEnE,2BAA2B;IAC3B,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAC7B,CAAC,eAAe,GAAG,EAAE;QACnB,gBAAgB,GAAG,EAAE;QACrB,CAAC,CAAC,GAAG,iBAAiB,CAAC,GAAG,EAAE;QAC5B,eAAe,GAAG,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,CACpC,CAAC;IAEF,OAAO;QACL,QAAQ;QACR,KAAK;QACL,MAAM;QACN,WAAW,EAAE,MAAM,CAAC,MAAM;QAC1B,eAAe;QACf,qBAAqB;QACrB,YAAY;QACZ,cAAc,EAAE,YAAY;QAC5B,SAAS;QACT,UAAU,EAAE;YACV,eAAe;YACf,iBAAiB,EAAE,gBAAgB;YACnC,iBAAiB;YACjB,eAAe;YACf,YAAY;YACZ,aAAa;SACd;KACF,CAAC;AACJ,CAAC;AAED,0DAA0D;AAC1D,oBAAoB;AACpB,0DAA0D;AAE1D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAkB,EAClB,KAAqB,EACrB,MAAwB,EACxB,GAAe;IAEf,MAAM,MAAM,GAAG,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAE/D,8BAA8B;IAC9B,MAAM,QAAQ,GAAc,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,YAAY,EAAE,CAAC,CAAC,WAAW,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,EAAE;KAClE,CAAC,CAAC,CAAC;IAEJ,MAAM,QAAQ,GAAiB;QAC7B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,sBAAsB,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE;KACxE,CAAC;IAEF,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,IAAI,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC;IACrC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,MAAM,CAAC,SAAS,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE,CAAC;QACtE,mBAAmB;QACnB,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;QAEzD,IAAI,CAAC;YACH,gCAAgC;YAChC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAChE,YAAY,IAAI,QAAQ,CAAC,SAAS,CAAC;YACnC,SAAS,IAAI,QAAQ,CAAC,IAAI,CAAC;YAE3B,mBAAmB;YACnB,IAAI,MAAM,CAAC,OAAO,IAAI,SAAS,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAClD,MAAM;YACR,CAAC;YAED,qBAAqB;YACrB,MAAM,WAAW,GAAiB,EAAE,CAAC;YAErC,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,KAAK,MAAM,QAAQ,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;oBAC1C,uCAAuC;oBACvC,QAAQ,CAAC,IAAI,CAAC;wBACZ,IAAI,EAAE,WAAW;wBACjB,OAAO,EAAE,IAAI;wBACb,UAAU,EAAE,CAAC;gCACX,EAAE,EAAE,QAAQ,KAAK,IAAI,QAAQ,CAAC,QAAQ,EAAE;gCACxC,IAAI,EAAE,UAAU;gCAChB,QAAQ,EAAE;oCACR,IAAI,EAAE,QAAQ,CAAC,QAAQ;oCACvB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC;iCAC9C;6BACF,CAAC;qBACH,CAAC,CAAC;oBAEH,8BAA8B;oBAC9B,MAAM,UAAU,GAAG,MAAM,CAAC,iBAAiB;wBACzC,CAAC,CAAC,oBAAoB,CAAC,QAAQ,EAAE,KAAK,CAAC;wBACvC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC,CAAC;oBAErE,WAAW,CAAC,IAAI,CAAC;wBACf,QAAQ,EAAE,QAAQ,CAAC,QAAQ;wBAC3B,KAAK,EAAE,QAAQ,CAAC,SAAS;wBACzB,MAAM,EAAE,UAAU;wBAClB,OAAO,EAAE,IAAI;qBACd,CAAC,CAAC;oBAEH,kCAAkC;oBAClC,QAAQ,CAAC,IAAI,CAAC;wBACZ,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,UAAU;wBACnB,YAAY,EAAE,QAAQ,KAAK,IAAI,QAAQ,CAAC,QAAQ,EAAE;qBACnD,CAAC,CAAC;gBACL,CAAC;gBAED,iCAAiC;gBACjC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBAChE,YAAY,IAAI,QAAQ,CAAC,SAAS,CAAC;gBACnC,SAAS,IAAI,QAAQ,CAAC,IAAI,CAAC;gBAE3B,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;gBAE7D,iCAAiC;gBACjC,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACpC,UAAU,GAAG,IAAI,CAAC;gBACpB,CAAC;gBAED,MAAM,CAAC,IAAI,CAAC;oBACV,WAAW,EAAE,KAAK;oBAClB,WAAW,EAAE,cAAc;oBAC3B,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE;oBACrE,WAAW;oBACX,UAAU,EAAE,aAAa,CAAC,KAAK,EAAE,QAAQ,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC;iBACtE,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,iCAAiC;gBACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;gBAE7D,MAAM,CAAC,IAAI,CAAC;oBACV,WAAW,EAAE,KAAK;oBAClB,WAAW,EAAE,cAAc;oBAC3B,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE;oBACrD,WAAW,EAAE,EAAE;oBACf,UAAU,EAAE,aAAa,CAAC,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC;iBACtD,CAAC,CAAC;gBAEH,6DAA6D;gBAC7D,gDAAgD;gBAChD,UAAU,GAAG,IAAI,CAAC;YACpB,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,+BAA+B;YAC/B,MAAM,CAAC,IAAI,CAAC;gBACV,WAAW,EAAE,KAAK;gBAClB,WAAW,EAAE,cAAc;gBAC3B,aAAa,EAAE,EAAE,IAAI,EAAE,UAAU,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE;gBAC5G,WAAW,EAAE,EAAE;gBACf,UAAU,EAAE;oBACV,mBAAmB,EAAE,IAAI;oBACzB,iBAAiB,EAAE,CAAC;oBACpB,kBAAkB,EAAE,EAAE;oBACtB,qBAAqB,EAAE,EAAE;oBACzB,gBAAgB,EAAE,KAAK;iBACxB;aACF,CAAC,CAAC;YACH,UAAU,GAAG,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,qBAAqB,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC;AACxF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAqB,EACrB,KAAqB,EACrB,MAAwB,EACxB,UAAuD;IAEvD,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,MAAM,GAAG,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IACrD,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,mBAAmB;QACnB,IAAI,MAAM,CAAC,OAAO,IAAI,SAAS,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAClD,MAAM;QACR,CAAC;QAED,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC9B,IAAI,CAAC,QAAQ;YAAE,SAAS;QAExB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;QACjE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;QAE9B,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC,CAAC,GAAG,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,eAAe,aAAa,CAAC"}

package/dist/agent/features/index.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Agent Features Module
+ *
+ * Exports all agent simulation features.
+ */
+export { generateTestCases, type TestCase, type TestCaseCategory, type TestCaseGenerationConfig, } from './test-case-generator.js';
+export { runSimulation, runSimulations, type SimulationConfig, type SimulationResult, type SimulationRound, type ToolResult, type RoundEvaluation, } from './agent-simulator.js';
+export { judgeSimulation, judgeSimulations, type SimulationJudgment, type JudgmentResult, type KeyMoment, type ToolImprovement, type AggregateJudgment, type FailurePattern, } from './simulation-judge.js';

package/dist/agent/features/index.js

@@ -0,0 +1,9 @@
+/**
+ * Agent Features Module
+ *
+ * Exports all agent simulation features.
+ */
+export { generateTestCases, } from './test-case-generator.js';
+export { runSimulation, runSimulations, } from './agent-simulator.js';
+export { judgeSimulation, judgeSimulations, } from './simulation-judge.js';
+//# sourceMappingURL=index.js.map

package/dist/agent/features/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/agent/features/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACL,iBAAiB,GAIlB,MAAM,0BAA0B,CAAC;AAElC,OAAO,EACL,aAAa,EACb,cAAc,GAMf,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,eAAe,EACf,gBAAgB,GAOjB,MAAM,uBAAuB,CAAC"}

package/dist/agent/features/simulation-judge.d.ts

@@ -0,0 +1,78 @@
+/**
+ * Simulation Judge
+ *
+ * Uses LLM to evaluate simulation results and provide structured scoring
+ * with actionable improvement suggestions.
+ */
+import { LLMClient } from '../llm-client.js';
+import type { DetectedTool } from '../../core/types/tool.js';
+import type { SimulationResult } from './agent-simulator.js';
+export interface JudgmentResult {
+    toolSelection: {
+        score: number;
+        reasoning: string;
+    };
+    parameterAccuracy: {
+        score: number;
+        reasoning: string;
+    };
+    conversationEfficiency: {
+        score: number;
+        reasoning: string;
+    };
+    errorHandling: {
+        score: number;
+        reasoning: string;
+    };
+    userExperience: {
+        score: number;
+        reasoning: string;
+    };
+    toolDefinitionQuality: {
+        score: number;
+        reasoning: string;
+    };
+}
+export interface SimulationJudgment {
+    scores: JudgmentResult;
+    overallScore: number;
+    verdict: 'PASS' | 'PARTIAL' | 'FAIL';
+    keyMoments: KeyMoment[];
+    toolImprovements: ToolImprovement[];
+    summary: string;
+}
+export interface KeyMoment {
+    moment: string;
+    impact: 'positive' | 'negative' | 'neutral';
+    severity: 'minor' | 'major' | 'critical';
+    toolFix?: string;
+}
+export interface ToolImprovement {
+    tool: string;
+    issue: string;
+    suggestedFix: string;
+    estimatedImpact: string;
+}
+export interface AggregateJudgment {
+    totalSimulations: number;
+    passRate: number;
+    avgScore: number;
+    failurePatterns: FailurePattern[];
+    topImprovements: ToolImprovement[];
+    summary: string;
+}
+export interface FailurePattern {
+    pattern: string;
+    frequency: number;
+    affectedTools: string[];
+    suggestedFix: string;
+}
+/**
+ * Judge a single simulation result
+ */
+export declare function judgeSimulation(simulation: SimulationResult, tools: DetectedTool[], llm?: LLMClient): Promise<SimulationJudgment>;
+/**
+ * Judge multiple simulations and provide aggregate analysis
+ */
+export declare function judgeSimulations(simulations: SimulationResult[], tools: DetectedTool[], llm?: LLMClient): Promise<AggregateJudgment>;
+export default judgeSimulation;